From 199fd2d074a7d02c5f5ddad205ca59032591d519 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Wed, 7 Dec 2022 15:08:33 +0100 Subject: [PATCH 01/20] +Remote Pairlist --- freqtrade/constants.py | 2 +- freqtrade/plugins/pairlist/RemotePairlist.py | 152 +++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 freqtrade/plugins/pairlist/RemotePairlist.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d869b89f6..dba277916 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'CalmarHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairlist', 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] diff --git a/freqtrade/plugins/pairlist/RemotePairlist.py b/freqtrade/plugins/pairlist/RemotePairlist.py new file mode 100644 index 000000000..3b1b56069 --- /dev/null +++ b/freqtrade/plugins/pairlist/RemotePairlist.py @@ -0,0 +1,152 @@ +""" +Remote PairList provider + +Provides dynamic pair list based on trade volumes +""" +import json +import logging +from typing import Any, Dict, List + +import requests +from cachetools import TTLCache + +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class RemotePairlist(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Config, pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if 'number_assets' not in self._pairlistconfig: + raise OperationalException( + '`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') + + self._number_pairs = self._pairlistconfig['number_assets'] + self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) + self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._pairlist_url = self._pairlistconfig.get('pairlist_url', + 'http://pairlist.robot.co.network') + self._stake_currency = config['stake_currency'] + + if (self._refresh_period < 850): + raise OperationalException( + 'Please set a Refresh Period higher than 850 for the Remotepairlist.' + ) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from Remote." + + def gen_pairlist(self, tickers: Tickers) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: List of pairs + """ + hick = "'" + double = '"' + # Generate dynamic whitelist + # Must always run if this pairlist is not the first in the list. + pairlist = self._pair_cache.get('pairlist') + + if pairlist: + # Item found - no refresh necessary + return pairlist.copy() + else: + + headers = { + 'User-Agent': 'Freqtrade Pairlist Fetcher', + } + + if "limit" not in self._pairlist_url: + url = self._pairlist_url + "&limit=" + str(self._number_pairs) + else: + url = self._pairlist_url + + if "stake" not in self._pairlist_url: + url = self._pairlist_url + "&stake=" + str(self._config['stake_currency']) + else: + url = self._pairlist_url + + if "exchange" not in self._pairlist_url: + url = self._pairlist_url + "&exchange=" + str(self._config['exchange']) + else: + url = self._pairlist_url + + try: + response = requests.get(url, headers=headers, timeout=60) + responser = response.text.replace(hick, double) + time_elapsed = response.elapsed.total_seconds() + rsplit = responser.split("#") + plist = rsplit[0].strip() + plist = plist.replace("
", "") + plist = json.loads(plist) + info = rsplit[1].strip() + + except Exception as e: + print(e) + self.log_once(f'Was not able to receive pairlist from' + f' {self._pairlist_url}', logger.info) + + if self._keep_pairlist_on_failure: + plist = pairlist + else: + plist = "" + + + pairlist = [] + + for i in plist: + if i not in pairlist: + if "/" in i: + if self._stake_currency in i: + pairlist.append(i) + else: + continue + else: + pairlist.append(i + "/" + self._config['stake_currency']) + + pairlist = self.filter_pairlist(pairlist, tickers) + self._pair_cache['pairlist'] = pairlist.copy() + self.log_once(info + " | " + "Fetched in " + str(time_elapsed) + " seconds.", logger.info) + return pairlist + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new whitelist + """ + + # Validate whitelist to only have active market pairs + pairlist = self._whitelist_for_active_markets(pairlist) + pairlist = self.verify_blacklist(pairlist, logger.info) + # Limit pairlist to the requested number of pairs + pairlist = pairlist[:self._number_pairs] + self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info) + + return pairlist From 48160f3fe9b099aa0c286fc78efcc5971186a323 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Wed, 7 Dec 2022 17:01:45 +0100 Subject: [PATCH 02/20] Flake 8 fix, Json Fetching --- freqtrade/constants.py | 2 +- freqtrade/plugins/pairlist/RemotePairList.py | 146 +++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 freqtrade/plugins/pairlist/RemotePairList.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index dba277916..e2eccfed3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'CalmarHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairlist', +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList', 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py new file mode 100644 index 000000000..684e68a1b --- /dev/null +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -0,0 +1,146 @@ +""" +Remote PairList provider + +Provides pair list fetched from a remote source +""" +import json +import logging +from typing import Any, Dict, List + +import requests +from cachetools import TTLCache + +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class RemotePairList(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Config, pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if 'number_assets' not in self._pairlistconfig: + raise OperationalException( + '`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') + + if 'pairlist_url' not in self._pairlistconfig: + raise OperationalException( + '`pairlist_url` not specified. Please check your configuration ' + 'for "pairlist.config.pairlist_url"') + + self._number_pairs = self._pairlistconfig['number_assets'] + self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) + self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._pairlist_url = self._pairlistconfig.get('pairlist_url', '') + self._read_timeout = self._pairlistconfig.get('read_timeout', 60) + self._last_pairlist: List[Any] = list() + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + + def gen_pairlist(self, tickers: Tickers) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: List of pairs + """ + pairlist = self._pair_cache.get('pairlist') + info = "" + + if pairlist: + # Item found - no refresh necessary + return pairlist.copy() + else: + # Fetch Pairlist from Remote + headers = { + 'User-Agent': 'Freqtrade - Remotepairlist', + } + + try: + response = requests.get(self._pairlist_url, headers=headers, + timeout=self._read_timeout) + content_type = response.headers.get('content-type') + time_elapsed = response.elapsed.total_seconds() + + rsplit = response.text.split("#") + + if "text/html" in str(content_type): + if len(rsplit) > 1: + plist = rsplit[0].strip() + plist = json.loads(plist) + info = rsplit[1].strip() + else: + plist = json.loads(rsplit[0]) + elif "application/json" in str(content_type): + jsonp = json.loads(' '.join(rsplit)) + plist = jsonp['pairs'] + info = jsonp['info'] + + except requests.exceptions.RequestException: + self.log_once(f'Was not able to fetch pairlist from:' + f' {self._pairlist_url}', logger.info) + + if self._keep_pairlist_on_failure: + plist = str(self._last_pairlist) + self.log_once('Keeping last fetched pairlist', logger.info) + else: + plist = "" + + time_elapsed = 0 + + pairlist = [] + + for i in plist: + if i not in pairlist: + pairlist.append(i) + else: + continue + + pairlist = self.filter_pairlist(pairlist, tickers) + self._pair_cache['pairlist'] = pairlist.copy() + + if(time_elapsed): + self.log_once(info + " | " + " Fetched in " + str(time_elapsed) + + " seconds.", logger.info) + + self._last_pairlist = list(pairlist) + return pairlist + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new whitelist + """ + + # Validate whitelist to only have active market pairs + pairlist = self._whitelist_for_active_markets(pairlist) + pairlist = self.verify_blacklist(pairlist, logger.info) + # Limit pairlist to the requested number of pairs + pairlist = pairlist[:self._number_pairs] + self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info) + + return pairlist From 607d5b2f8f0e870c34fe3bdee2c8fe6cff4af37c Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Wed, 7 Dec 2022 17:47:38 +0100 Subject: [PATCH 03/20] Split to fetch_pairlist function, Info Message --- freqtrade/plugins/pairlist/RemotePairList.py | 87 +++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 684e68a1b..b6d0abe35 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -59,6 +59,49 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + def fetch_pairlist(self): + headers = { + 'User-Agent': 'Freqtrade - Remotepairlist', + } + + try: + response = requests.get(self._pairlist_url, headers=headers, + timeout=self._read_timeout) + content_type = response.headers.get('content-type') + time_elapsed = response.elapsed.total_seconds() + + rsplit = response.text.split("#") + + if "text/html" in str(content_type): + if len(rsplit) > 1: + plist = rsplit[0].strip() + plist = json.loads(plist) + info = rsplit[1].strip() + else: + plist = json.loads(rsplit[0]) + elif "application/json" in str(content_type): + jsonr = response.json() + plist = jsonr['pairs'] + + if 'info' in jsonr: + info = jsonr['info'] + if 'refresh_period' in jsonr: + self._refresh_period = jsonr['refresh_period'] + + except requests.exceptions.RequestException: + self.log_once(f'Was not able to fetch pairlist from:' + f' {self._pairlist_url}', logger.info) + + if self._keep_pairlist_on_failure: + plist = str(self._last_pairlist) + self.log_once('Keeping last fetched pairlist', logger.info) + else: + plist = "" + + time_elapsed = 0 + + return plist, time_elapsed, info + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist @@ -66,49 +109,14 @@ class RemotePairList(IPairList): :return: List of pairs """ pairlist = self._pair_cache.get('pairlist') - info = "" + info = "Pairlist" if pairlist: # Item found - no refresh necessary return pairlist.copy() else: - # Fetch Pairlist from Remote - headers = { - 'User-Agent': 'Freqtrade - Remotepairlist', - } - - try: - response = requests.get(self._pairlist_url, headers=headers, - timeout=self._read_timeout) - content_type = response.headers.get('content-type') - time_elapsed = response.elapsed.total_seconds() - - rsplit = response.text.split("#") - - if "text/html" in str(content_type): - if len(rsplit) > 1: - plist = rsplit[0].strip() - plist = json.loads(plist) - info = rsplit[1].strip() - else: - plist = json.loads(rsplit[0]) - elif "application/json" in str(content_type): - jsonp = json.loads(' '.join(rsplit)) - plist = jsonp['pairs'] - info = jsonp['info'] - - except requests.exceptions.RequestException: - self.log_once(f'Was not able to fetch pairlist from:' - f' {self._pairlist_url}', logger.info) - - if self._keep_pairlist_on_failure: - plist = str(self._last_pairlist) - self.log_once('Keeping last fetched pairlist', logger.info) - else: - plist = "" - - time_elapsed = 0 - + # Fetch Pairlist from Remote URL + plist, time_elapsed, info = self.fetch_pairlist() pairlist = [] for i in plist: @@ -121,8 +129,7 @@ class RemotePairList(IPairList): self._pair_cache['pairlist'] = pairlist.copy() if(time_elapsed): - self.log_once(info + " | " + " Fetched in " + str(time_elapsed) + - " seconds.", logger.info) + self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) self._last_pairlist = list(pairlist) return pairlist From 547a75d9c1abc42db10b811c152147a66d48a6af Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Wed, 7 Dec 2022 17:49:21 +0100 Subject: [PATCH 04/20] Fix Info --- freqtrade/plugins/pairlist/RemotePairList.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index b6d0abe35..07829d246 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -64,6 +64,8 @@ class RemotePairList(IPairList): 'User-Agent': 'Freqtrade - Remotepairlist', } + info = "Pairlist" + try: response = requests.get(self._pairlist_url, headers=headers, timeout=self._read_timeout) @@ -109,7 +111,6 @@ class RemotePairList(IPairList): :return: List of pairs """ pairlist = self._pair_cache.get('pairlist') - info = "Pairlist" if pairlist: # Item found - no refresh necessary From b144a6357d7cbafa1ab7ded091f6e5ad79a78027 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Wed, 7 Dec 2022 18:24:55 +0100 Subject: [PATCH 05/20] Remove Duplicate --- freqtrade/plugins/pairlist/RemotePairlist.py | 152 ------------------- 1 file changed, 152 deletions(-) delete mode 100644 freqtrade/plugins/pairlist/RemotePairlist.py diff --git a/freqtrade/plugins/pairlist/RemotePairlist.py b/freqtrade/plugins/pairlist/RemotePairlist.py deleted file mode 100644 index 3b1b56069..000000000 --- a/freqtrade/plugins/pairlist/RemotePairlist.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Remote PairList provider - -Provides dynamic pair list based on trade volumes -""" -import json -import logging -from typing import Any, Dict, List - -import requests -from cachetools import TTLCache - -from freqtrade.constants import Config -from freqtrade.exceptions import OperationalException -from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList - - -logger = logging.getLogger(__name__) - - -class RemotePairlist(IPairList): - - def __init__(self, exchange, pairlistmanager, - config: Config, pairlistconfig: Dict[str, Any], - pairlist_pos: int) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - - if 'number_assets' not in self._pairlistconfig: - raise OperationalException( - '`number_assets` not specified. Please check your configuration ' - 'for "pairlist.config.number_assets"') - - self._number_pairs = self._pairlistconfig['number_assets'] - self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) - self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) - self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) - self._pairlist_url = self._pairlistconfig.get('pairlist_url', - 'http://pairlist.robot.co.network') - self._stake_currency = config['stake_currency'] - - if (self._refresh_period < 850): - raise OperationalException( - 'Please set a Refresh Period higher than 850 for the Remotepairlist.' - ) - - @property - def needstickers(self) -> bool: - """ - Boolean property defining if tickers are necessary. - If no Pairlist requires tickers, an empty Dict is passed - as tickers argument to filter_pairlist - """ - return False - - def short_desc(self) -> str: - """ - Short whitelist method description - used for startup-messages - """ - return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from Remote." - - def gen_pairlist(self, tickers: Tickers) -> List[str]: - """ - Generate the pairlist - :param tickers: Tickers (from exchange.get_tickers). May be cached. - :return: List of pairs - """ - hick = "'" - double = '"' - # Generate dynamic whitelist - # Must always run if this pairlist is not the first in the list. - pairlist = self._pair_cache.get('pairlist') - - if pairlist: - # Item found - no refresh necessary - return pairlist.copy() - else: - - headers = { - 'User-Agent': 'Freqtrade Pairlist Fetcher', - } - - if "limit" not in self._pairlist_url: - url = self._pairlist_url + "&limit=" + str(self._number_pairs) - else: - url = self._pairlist_url - - if "stake" not in self._pairlist_url: - url = self._pairlist_url + "&stake=" + str(self._config['stake_currency']) - else: - url = self._pairlist_url - - if "exchange" not in self._pairlist_url: - url = self._pairlist_url + "&exchange=" + str(self._config['exchange']) - else: - url = self._pairlist_url - - try: - response = requests.get(url, headers=headers, timeout=60) - responser = response.text.replace(hick, double) - time_elapsed = response.elapsed.total_seconds() - rsplit = responser.split("#") - plist = rsplit[0].strip() - plist = plist.replace("
", "") - plist = json.loads(plist) - info = rsplit[1].strip() - - except Exception as e: - print(e) - self.log_once(f'Was not able to receive pairlist from' - f' {self._pairlist_url}', logger.info) - - if self._keep_pairlist_on_failure: - plist = pairlist - else: - plist = "" - - - pairlist = [] - - for i in plist: - if i not in pairlist: - if "/" in i: - if self._stake_currency in i: - pairlist.append(i) - else: - continue - else: - pairlist.append(i + "/" + self._config['stake_currency']) - - pairlist = self.filter_pairlist(pairlist, tickers) - self._pair_cache['pairlist'] = pairlist.copy() - self.log_once(info + " | " + "Fetched in " + str(time_elapsed) + " seconds.", logger.info) - return pairlist - - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: - """ - Filters and sorts pairlist and returns the whitelist again. - Called on each bot iteration - please use internal caching if necessary - :param pairlist: pairlist to filter or sort - :param tickers: Tickers (from exchange.get_tickers). May be cached. - :return: new whitelist - """ - - # Validate whitelist to only have active market pairs - pairlist = self._whitelist_for_active_markets(pairlist) - pairlist = self.verify_blacklist(pairlist, logger.info) - # Limit pairlist to the requested number of pairs - pairlist = pairlist[:self._number_pairs] - self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info) - - return pairlist From da2747d487ced9129a3b3ae8336e6d7533da5132 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Thu, 8 Dec 2022 00:52:54 +0100 Subject: [PATCH 06/20] Add Local .json file Loading --- freqtrade/plugins/pairlist/RemotePairList.py | 30 +++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 07829d246..c3b612067 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -5,6 +5,7 @@ Provides pair list fetched from a remote source """ import json import logging +from pathlib import Path from typing import Any, Dict, List import requests @@ -110,21 +111,36 @@ class RemotePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: List of pairs """ + + time_elapsed = 0 pairlist = self._pair_cache.get('pairlist') if pairlist: # Item found - no refresh necessary return pairlist.copy() else: - # Fetch Pairlist from Remote URL - plist, time_elapsed, info = self.fetch_pairlist() - pairlist = [] + if self._pairlist_url.startswith("file:///"): + filename = self._pairlist_url.split("file:///", 1)[1] + file_path = Path(filename) - for i in plist: - if i not in pairlist: - pairlist.append(i) + if file_path.exists(): + with open(filename) as json_file: + # Load the JSON data into a dictionary + jsonp = json.load(json_file) + plist = jsonp['pairs'] else: - continue + raise ValueError(f"{self._pairlist_url} does not exist.") + else: + # Fetch Pairlist from Remote URL + plist, time_elapsed, info = self.fetch_pairlist() + + pairlist = [] + + for i in plist: + if i not in pairlist: + pairlist.append(i) + else: + continue pairlist = self.filter_pairlist(pairlist, tickers) self._pair_cache['pairlist'] = pairlist.copy() From 7efcbbb4573c3a5ff75cca0fc892cdc6a743e779 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Thu, 8 Dec 2022 01:09:17 +0100 Subject: [PATCH 07/20] Local File Loading --- freqtrade/plugins/pairlist/RemotePairList.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index c3b612067..af8b67577 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -83,13 +83,11 @@ class RemotePairList(IPairList): else: plist = json.loads(rsplit[0]) elif "application/json" in str(content_type): - jsonr = response.json() - plist = jsonr['pairs'] + jsonp = response.json() + plist = jsonp['pairs'] - if 'info' in jsonr: - info = jsonr['info'] - if 'refresh_period' in jsonr: - self._refresh_period = jsonr['refresh_period'] + info = jsonp.get('info', '') + self._refresh_period = jsonp.get('refresh_period', self._refresh_period) except requests.exceptions.RequestException: self.log_once(f'Was not able to fetch pairlist from:' @@ -128,6 +126,10 @@ class RemotePairList(IPairList): # Load the JSON data into a dictionary jsonp = json.load(json_file) plist = jsonp['pairs'] + + info = jsonp.get('info', '') + self._refresh_period = jsonp.get('refresh_period', self._refresh_period) + else: raise ValueError(f"{self._pairlist_url} does not exist.") else: @@ -145,8 +147,10 @@ class RemotePairList(IPairList): pairlist = self.filter_pairlist(pairlist, tickers) self._pair_cache['pairlist'] = pairlist.copy() - if(time_elapsed): + if (time_elapsed) in locals(): self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) + else: + self.log_once(f'{info} Fetched Pairlist.', logger.info) self._last_pairlist = list(pairlist) return pairlist From 66412bfa58645177ebcef18c4c8ecf4a875527c2 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Thu, 8 Dec 2022 01:51:12 +0100 Subject: [PATCH 08/20] Remove unnecessary loop --- freqtrade/plugins/pairlist/RemotePairList.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index af8b67577..7367f713c 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -110,8 +110,8 @@ class RemotePairList(IPairList): :return: List of pairs """ - time_elapsed = 0 pairlist = self._pair_cache.get('pairlist') + time_elapsed = 0 if pairlist: # Item found - no refresh necessary @@ -125,7 +125,7 @@ class RemotePairList(IPairList): with open(filename) as json_file: # Load the JSON data into a dictionary jsonp = json.load(json_file) - plist = jsonp['pairs'] + pairlist = jsonp['pairs'] info = jsonp.get('info', '') self._refresh_period = jsonp.get('refresh_period', self._refresh_period) @@ -134,20 +134,12 @@ class RemotePairList(IPairList): raise ValueError(f"{self._pairlist_url} does not exist.") else: # Fetch Pairlist from Remote URL - plist, time_elapsed, info = self.fetch_pairlist() - - pairlist = [] - - for i in plist: - if i not in pairlist: - pairlist.append(i) - else: - continue + pairlist, time_elapsed, info = self.fetch_pairlist() pairlist = self.filter_pairlist(pairlist, tickers) self._pair_cache['pairlist'] = pairlist.copy() - if (time_elapsed) in locals(): + if time_elapsed: self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) else: self.log_once(f'{info} Fetched Pairlist.', logger.info) From f6b90595fae9a24cd0f2a3a3e83d824bf597e129 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Mon, 12 Dec 2022 11:05:03 +0100 Subject: [PATCH 09/20] remove html. change var names. --- freqtrade/plugins/pairlist/RemotePairList.py | 53 +++++++++----------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 7367f713c..ef5463a56 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -6,7 +6,7 @@ Provides pair list fetched from a remote source import json import logging from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple import requests from cachetools import TTLCache @@ -60,7 +60,7 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." - def fetch_pairlist(self): + def fetch_pairlist(self) -> Tuple[List[str], float, str]: headers = { 'User-Agent': 'Freqtrade - Remotepairlist', } @@ -68,40 +68,35 @@ class RemotePairList(IPairList): info = "Pairlist" try: - response = requests.get(self._pairlist_url, headers=headers, - timeout=self._read_timeout) - content_type = response.headers.get('content-type') - time_elapsed = response.elapsed.total_seconds() + with requests.get(self._pairlist_url, headers=headers, + timeout=self._read_timeout) as response: + content_type = response.headers.get('content-type') + time_elapsed = response.elapsed.total_seconds() - rsplit = response.text.split("#") - - if "text/html" in str(content_type): - if len(rsplit) > 1: - plist = rsplit[0].strip() - plist = json.loads(plist) - info = rsplit[1].strip() + if "application/json" in str(content_type): + jsonparse = response.json() + pairlist = jsonparse['pairs'] + info = jsonparse.get('info', '') else: - plist = json.loads(rsplit[0]) - elif "application/json" in str(content_type): - jsonp = response.json() - plist = jsonp['pairs'] + raise OperationalException( + 'Remotepairlist is not of type JSON abort') - info = jsonp.get('info', '') - self._refresh_period = jsonp.get('refresh_period', self._refresh_period) + self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) except requests.exceptions.RequestException: self.log_once(f'Was not able to fetch pairlist from:' f' {self._pairlist_url}', logger.info) if self._keep_pairlist_on_failure: - plist = str(self._last_pairlist) + pairlist = self._last_pairlist self.log_once('Keeping last fetched pairlist', logger.info) else: - plist = "" + pairlist = [] time_elapsed = 0 - return plist, time_elapsed, info + return pairlist, time_elapsed, info def gen_pairlist(self, tickers: Tickers) -> List[str]: """ @@ -111,7 +106,7 @@ class RemotePairList(IPairList): """ pairlist = self._pair_cache.get('pairlist') - time_elapsed = 0 + time_elapsed = 0.0 if pairlist: # Item found - no refresh necessary @@ -124,11 +119,11 @@ class RemotePairList(IPairList): if file_path.exists(): with open(filename) as json_file: # Load the JSON data into a dictionary - jsonp = json.load(json_file) - pairlist = jsonp['pairs'] - - info = jsonp.get('info', '') - self._refresh_period = jsonp.get('refresh_period', self._refresh_period) + jsonparse = json.load(json_file) + pairlist = jsonparse['pairs'] + info = jsonparse.get('info', '') + self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) else: raise ValueError(f"{self._pairlist_url} does not exist.") @@ -139,7 +134,7 @@ class RemotePairList(IPairList): pairlist = self.filter_pairlist(pairlist, tickers) self._pair_cache['pairlist'] = pairlist.copy() - if time_elapsed: + if time_elapsed != 0.0: self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) else: self.log_once(f'{info} Fetched Pairlist.', logger.info) From 6f92c58e3317773a2ffedaf33e9f465d358ec528 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Mon, 12 Dec 2022 13:24:33 +0100 Subject: [PATCH 10/20] add docs, add bearer token. --- docs/includes/pairlists.md | 46 +++++++++++++++++++- freqtrade/plugins/pairlist/RemotePairList.py | 12 +++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index d61718c7d..c12683e75 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -2,7 +2,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. They are configured in the `pairlists` section of the configuration settings. -In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). +In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler), Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. @@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) * [`ProducerPairList`](#producerpairlist) +* [`RemotePairList`](#remotepairlist) * [`AgeFilter`](#agefilter) * [`OffsetFilter`](#offsetfilter) * [`PerformanceFilter`](#performancefilter) @@ -173,6 +174,49 @@ You can limit the length of the pairlist with the optional parameter `number_ass `ProducerPairList` can also be used multiple times in sequence, combining the pairs from multiple producers. Obviously in complex such configurations, the Producer may not provide data for all pairs, so the strategy must be fit for this. +#### RemotePairList + +It allows the user to fetch a pairlist from a remote server or a locally stored json file within the freqtrade directory, enabling dynamic updates and customization of the trading pairlist. + +The RemotePairList is defined in the pairlists section of the configuration settings. It uses the following configuration options: + +```json +"pairlists": [ + { + "method": "RemotePairList", + "pairlist_url": "https://example.com/pairlist", + "number_assets": 10, + "refresh_period": 1800, + "keep_pairlist_on_failure": true, + "read_timeout": 60, + "bearer_token": "my-bearer-token" + } +] +``` + +The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist. + +The user is responsible for providing a server or local file that returns a JSON object with the following structure: + +```json +{ + "pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"], + "refresh_period": 1800, + "info": "Pairlist updated on 2022-12-12 at 12:12" +} +``` + +The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed. The `info` property is also optional and can be used to provide any additional information about the pairlist. + +The optional `keep_pairlist_on_failure` specifies whether the previous received pairlist should be used if the remote server is not reachable or returns an error. The default value is true. + +The optional `read_timeout` specifies the maximum amount of time (in seconds) to wait for a response from the remote source, The default value is 60. + +The optional `bearer_token` will be included in the requests Authorization Header. + +!!! Note + In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned. + #### AgeFilter Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity). diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index ef5463a56..7ef038da7 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -11,6 +11,7 @@ from typing import Any, Dict, List, Tuple import requests from cachetools import TTLCache +from freqtrade import __version__ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers @@ -43,6 +44,7 @@ class RemotePairList(IPairList): self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) self._pairlist_url = self._pairlistconfig.get('pairlist_url', '') self._read_timeout = self._pairlistconfig.get('read_timeout', 60) + self._bearer_token = self._pairlistconfig.get('bearer_token', '') self._last_pairlist: List[Any] = list() @property @@ -61,10 +63,14 @@ class RemotePairList(IPairList): return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." def fetch_pairlist(self) -> Tuple[List[str], float, str]: + headers = { - 'User-Agent': 'Freqtrade - Remotepairlist', + 'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist' } + if self._bearer_token: + headers['Authorization'] = f'Bearer {self._bearer_token}' + info = "Pairlist" try: @@ -76,7 +82,7 @@ class RemotePairList(IPairList): if "application/json" in str(content_type): jsonparse = response.json() pairlist = jsonparse['pairs'] - info = jsonparse.get('info', '') + info = jsonparse.get('info', '')[:1000] else: raise OperationalException( 'Remotepairlist is not of type JSON abort') @@ -121,7 +127,7 @@ class RemotePairList(IPairList): # Load the JSON data into a dictionary jsonparse = json.load(json_file) pairlist = jsonparse['pairs'] - info = jsonparse.get('info', '') + info = jsonparse.get('info', '')[:1000] self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) From d52c1c75544aee98f06be81cbce74d2fb45500b5 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Tue, 13 Dec 2022 20:21:06 +0100 Subject: [PATCH 11/20] Add unit tests --- docs/includes/pairlists.md | 2 +- freqtrade/plugins/pairlist/RemotePairList.py | 29 ++--- tests/plugins/test_remotepairlist.py | 123 +++++++++++++++++++ 3 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 tests/plugins/test_remotepairlist.py diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index c12683e75..3a6ab7a3c 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -2,7 +2,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. They are configured in the `pairlists` section of the configuration settings. -In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler), Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). +In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 7ef038da7..418ac5b0b 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -74,21 +74,22 @@ class RemotePairList(IPairList): info = "Pairlist" try: - with requests.get(self._pairlist_url, headers=headers, - timeout=self._read_timeout) as response: - content_type = response.headers.get('content-type') - time_elapsed = response.elapsed.total_seconds() + response = requests.get(self._pairlist_url, headers=headers, + timeout=self._read_timeout) + content_type = response.headers.get('content-type') + time_elapsed = response.elapsed.total_seconds() - if "application/json" in str(content_type): - jsonparse = response.json() - pairlist = jsonparse['pairs'] - info = jsonparse.get('info', '')[:1000] - else: - raise OperationalException( - 'Remotepairlist is not of type JSON abort') + print(response) - self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + if "application/json" in str(content_type): + jsonparse = response.json() + pairlist = jsonparse['pairs'] + info = jsonparse.get('info', '') + else: + raise OperationalException('RemotePairList is not of type JSON abort ') + + self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) except requests.exceptions.RequestException: self.log_once(f'Was not able to fetch pairlist from:' @@ -127,7 +128,7 @@ class RemotePairList(IPairList): # Load the JSON data into a dictionary jsonparse = json.load(json_file) pairlist = jsonparse['pairs'] - info = jsonparse.get('info', '')[:1000] + info = jsonparse.get('info', '') self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py new file mode 100644 index 000000000..743534bc3 --- /dev/null +++ b/tests/plugins/test_remotepairlist.py @@ -0,0 +1,123 @@ +from unittest.mock import MagicMock + +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.RemotePairList import RemotePairList +from freqtrade.plugins.pairlistmanager import PairListManager +from tests.conftest import get_patched_exchange, get_patched_freqtradebot + + +@pytest.fixture(scope="function") +def rpl_config(default_conf): + default_conf['stake_currency'] = 'USDT' + + default_conf['exchange']['pair_whitelist'] = [ + 'ETH/USDT', + 'BTC/USDT', + ] + default_conf['exchange']['pair_blacklist'] = [ + 'BLK/USDT' + ] + return default_conf + + +def test_fetch_pairlist_mock_response_html(mocker, rpl_config): + mock_response = MagicMock() + mock_response.headers = {'content-type': 'text/html'} + mocker.patch('requests.get', return_value=mock_response) + + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "pairlist_url": "http://example.com/pairlist", + "number_assets": 10, + "read_timeout": 10, + "keep_pairlist_on_failure": True, + } + ] + + exchange = get_patched_exchange(mocker, rpl_config) + pairlistmanager = PairListManager(exchange, rpl_config) + + mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get", + return_value=mock_response) + remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, + rpl_config['pairlists'][0], 0) + + with pytest.raises(OperationalException, match='RemotePairList is not of type JSON abort'): + remote_pairlist.fetch_pairlist() + + +def test_remote_pairlist_init_no_pairlist_url(mocker, rpl_config): + + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "number_assets": 10, + "keep_pairlist_on_failure": True, + } + ] + + get_patched_exchange(mocker, rpl_config) + with pytest.raises(OperationalException, match=r'`pairlist_url` not specified.' + r' Please check your configuration for "pairlist.config.pairlist_url"'): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_remote_pairlist_init_no_number_assets(mocker, rpl_config): + + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "pairlist_url": "http://example.com/pairlist", + "keep_pairlist_on_failure": True, + } + ] + + get_patched_exchange(mocker, rpl_config) + + with pytest.raises(OperationalException, match=r'`number_assets` not specified. ' + 'Please check your configuration for "pairlist.config.number_assets"'): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): + + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "pairlist_url": "http://example.com/pairlist", + "number_assets": 10, + "refresh_period": 10, + "read_timeout": 10, + "keep_pairlist_on_failure": True, + } + ] + + mock_response = MagicMock() + + mock_response.json.return_value = { + "pairs": ["ETH/BTC", "XRP/BTC", "LTC/BTC", "EOS/BTC"], + "info": "Mock pairlist response", + "refresh_period": 60 + } + + mock_response.headers = { + "content-type": "application/json" + } + + mock_response.elapsed.total_seconds.return_value = 0.4 + mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get", + return_value=mock_response) + + exchange = get_patched_exchange(mocker, rpl_config) + pairlistmanager = PairListManager(exchange, rpl_config) + remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, + rpl_config['pairlists'][0], 0) + pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() + + assert pairs == ["ETH/BTC", "XRP/BTC", "LTC/BTC", "EOS/BTC"] + assert time_elapsed == 0.4 + assert info == "Mock pairlist response" + assert remote_pairlist._refresh_period == 60 From 7f3524949c17afa87f52d8023770d8a974884b72 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Tue, 13 Dec 2022 21:00:23 +0100 Subject: [PATCH 12/20] - print --- freqtrade/plugins/pairlist/RemotePairList.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 418ac5b0b..e46ac0419 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -79,8 +79,6 @@ class RemotePairList(IPairList): content_type = response.headers.get('content-type') time_elapsed = response.elapsed.total_seconds() - print(response) - if "application/json" in str(content_type): jsonparse = response.json() pairlist = jsonparse['pairs'] From 1d5c66da3bcb212732df322efb74d54eca069ca0 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Thu, 15 Dec 2022 17:38:21 +0100 Subject: [PATCH 13/20] + Unit Tests --- tests/plugins/test_remotepairlist.py | 72 ++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index 743534bc3..bc4adb616 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -1,11 +1,13 @@ +import json from unittest.mock import MagicMock import pytest +import requests from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.RemotePairList import RemotePairList from freqtrade.plugins.pairlistmanager import PairListManager -from tests.conftest import get_patched_exchange, get_patched_freqtradebot +from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has @pytest.fixture(scope="function") @@ -22,10 +24,44 @@ def rpl_config(default_conf): return default_conf +def test_gen_pairlist_with_local_file(mocker, rpl_config): + + mock_file = MagicMock() + mock_file.read.return_value = '{"pairs": ["TKN/USDT","ETH/USDT","NANO/USDT"]}' + mocker.patch('freqtrade.plugins.pairlist.RemotePairList.open', return_value=mock_file) + + mock_file_path = mocker.patch('freqtrade.plugins.pairlist.RemotePairList.Path') + mock_file_path.exists.return_value = True + + jsonparse = json.loads(mock_file.read.return_value) + mocker.patch('freqtrade.plugins.pairlist.RemotePairList.json.load', return_value=jsonparse) + + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + 'number_assets': 2, + 'refresh_period': 1800, + 'keep_pairlist_on_failure': True, + 'pairlist_url': 'file:///pairlist.json', + 'bearer_token': '', + 'read_timeout': 60 + } + ] + + exchange = get_patched_exchange(mocker, rpl_config) + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, + rpl_config['pairlists'][0], 0) + + result = remote_pairlist.gen_pairlist([]) + + assert result == ['TKN/USDT', 'ETH/USDT'] + + def test_fetch_pairlist_mock_response_html(mocker, rpl_config): mock_response = MagicMock() mock_response.headers = {'content-type': 'text/html'} - mocker.patch('requests.get', return_value=mock_response) rpl_config['pairlists'] = [ { @@ -49,6 +85,34 @@ def test_fetch_pairlist_mock_response_html(mocker, rpl_config): remote_pairlist.fetch_pairlist() +def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog): + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "pairlist_url": "http://example.com/pairlist", + "number_assets": 10, + "read_timeout": 10, + "keep_pairlist_on_failure": True, + } + ] + + exchange = get_patched_exchange(mocker, rpl_config) + pairlistmanager = PairListManager(exchange, rpl_config) + + mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get", + side_effect=requests.exceptions.RequestException) + + remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, + rpl_config['pairlists'][0], 0) + + remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"] + + pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() + assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url} ", caplog) + assert log_has("Keeping last fetched pairlist", caplog) + assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"] + + def test_remote_pairlist_init_no_pairlist_url(mocker, rpl_config): rpl_config['pairlists'] = [ @@ -98,7 +162,7 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): mock_response = MagicMock() mock_response.json.return_value = { - "pairs": ["ETH/BTC", "XRP/BTC", "LTC/BTC", "EOS/BTC"], + "pairs": ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"], "info": "Mock pairlist response", "refresh_period": 60 } @@ -117,7 +181,7 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): rpl_config['pairlists'][0], 0) pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() - assert pairs == ["ETH/BTC", "XRP/BTC", "LTC/BTC", "EOS/BTC"] + assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"] assert time_elapsed == 0.4 assert info == "Mock pairlist response" assert remote_pairlist._refresh_period == 60 From cd1b8b9cee37a0ac412a57af51f02348b40d9565 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Thu, 15 Dec 2022 18:14:37 +0100 Subject: [PATCH 14/20] single space removed for the unit test to pass.. --- tests/plugins/test_remotepairlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index bc4adb616..fc91d3f06 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -108,7 +108,7 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog): remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"] pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() - assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url} ", caplog) + assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog) assert log_has("Keeping last fetched pairlist", caplog) assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"] From 6fa3db3a1dc79dfd36b121c5b3f41cc8811ad487 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Dec 2022 19:36:21 +0100 Subject: [PATCH 15/20] Fix failing tests --- tests/plugins/test_pairlist.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index ecc1da3e3..739c3a7ac 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -22,6 +22,11 @@ from tests.conftest import (create_mock_trades_usdt, get_patched_exchange, get_p log_has, log_has_re, num_log_has) +# Exclude RemotePairList from tests. +# It has a mandatory parameter, and requires special handling, which happens in test_remotepairlist. +TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ['RemotePairList']] + + @pytest.fixture(scope="function") def whitelist_conf(default_conf): default_conf['stake_currency'] = 'BTC' @@ -824,7 +829,7 @@ def test_pair_whitelist_not_supported_Spread(mocker, default_conf, tickers) -> N get_patched_freqtradebot(mocker, default_conf) -@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) +@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS) def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): whitelist_conf['pairlists'][0]['method'] = pairlist mocker.patch.multiple('freqtrade.exchange.Exchange', @@ -839,7 +844,7 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): assert isinstance(freqtrade.pairlists.blacklist, list) -@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) +@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS) @pytest.mark.parametrize("whitelist,log_message", [ (['ETH/BTC', 'TKN/BTC'], ""), # TRX/ETH not in markets @@ -872,7 +877,7 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist assert log_message in caplog.text -@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) +@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS) def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, pairlist, tickers): whitelist_conf['pairlists'][0]['method'] = pairlist From bb33b96ba7bcf90f442251c9e7e4d44392cec9a2 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Sun, 18 Dec 2022 22:28:12 +0100 Subject: [PATCH 16/20] init cache on first iteration, init checks, limit length and charmap to info replace if invalid, move filter logic --- docs/includes/pairlists.md | 2 +- freqtrade/plugins/pairlist/RemotePairList.py | 87 +++++++++++++------- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 3a6ab7a3c..0bff9b29b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -202,7 +202,7 @@ The user is responsible for providing a server or local file that returns a JSON { "pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"], "refresh_period": 1800, - "info": "Pairlist updated on 2022-12-12 at 12:12" + "info": "Pairlist updated on 2022-12-12 at 12:12" // Maximum Length: 256 Characters, Charset: Alphanumeric + "+-.,%:" } ``` diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index e46ac0419..a0e140b42 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -6,7 +6,7 @@ Provides pair list fetched from a remote source import json import logging from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple import requests from cachetools import TTLCache @@ -39,12 +39,13 @@ class RemotePairList(IPairList): 'for "pairlist.config.pairlist_url"') self._number_pairs = self._pairlistconfig['number_assets'] - self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800) self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) - self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._pair_cache: Optional[TTLCache] = None self._pairlist_url = self._pairlistconfig.get('pairlist_url', '') self._read_timeout = self._pairlistconfig.get('read_timeout', 60) self._bearer_token = self._pairlistconfig.get('bearer_token', '') + self._init_done = False self._last_pairlist: List[Any] = list() @property @@ -62,6 +63,15 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + def return_last_pairlist(self) -> List[str]: + if self._keep_pairlist_on_failure: + pairlist = self._last_pairlist + self.log_once('Keeping last fetched pairlist', logger.info) + else: + pairlist = [] + + return pairlist + def fetch_pairlist(self) -> Tuple[List[str], float, str]: headers = { @@ -81,23 +91,35 @@ class RemotePairList(IPairList): if "application/json" in str(content_type): jsonparse = response.json() - pairlist = jsonparse['pairs'] - info = jsonparse.get('info', '') - else: - raise OperationalException('RemotePairList is not of type JSON abort ') + pairlist = jsonparse.get('pairs', []) + remote_info = jsonparse.get('info', '')[:256].strip() + remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period) - self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + info = "".join(char if char.isalnum() or + char in " +-.,%:" else "-" for char in remote_info) + + if not self._init_done and self._refresh_period < remote_refresh_period: + self.log_once(f'Refresh Period has been increased from {self._refresh_period}' + f' to {remote_refresh_period} from Remote.', logger.info) + + self._refresh_period = remote_refresh_period + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + + self._init_done = True + else: + if self._init_done: + self.log_once(f'Error: RemotePairList is not of type JSON: ' + f' {self._pairlist_url}', logger.info) + pairlist = self.return_last_pairlist() + + else: + raise OperationalException('RemotePairList is not of type JSON abort ') except requests.exceptions.RequestException: self.log_once(f'Was not able to fetch pairlist from:' f' {self._pairlist_url}', logger.info) - if self._keep_pairlist_on_failure: - pairlist = self._last_pairlist - self.log_once('Keeping last fetched pairlist', logger.info) - else: - pairlist = [] + pairlist = self.return_last_pairlist() time_elapsed = 0 @@ -110,12 +132,17 @@ class RemotePairList(IPairList): :return: List of pairs """ - pairlist = self._pair_cache.get('pairlist') + if self._init_done and self._pair_cache: + pairlist = self._pair_cache.get('pairlist') + else: + pairlist = [] + time_elapsed = 0.0 if pairlist: # Item found - no refresh necessary return pairlist.copy() + self._init_done = True else: if self._pairlist_url.startswith("file:///"): filename = self._pairlist_url.split("file:///", 1)[1] @@ -127,17 +154,25 @@ class RemotePairList(IPairList): jsonparse = json.load(json_file) pairlist = jsonparse['pairs'] info = jsonparse.get('info', '') - self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + if not self._init_done: + self._refresh_period = jsonparse.get('refresh_period', + self._refresh_period) + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._init_done = True else: raise ValueError(f"{self._pairlist_url} does not exist.") else: # Fetch Pairlist from Remote URL pairlist, time_elapsed, info = self.fetch_pairlist() - pairlist = self.filter_pairlist(pairlist, tickers) - self._pair_cache['pairlist'] = pairlist.copy() + self.log_once(f"Fetched pairs: {pairlist}", logger.debug) + + pairlist = self._whitelist_for_active_markets(pairlist) + pairlist = pairlist[:self._number_pairs] + + if self._pair_cache: + self._pair_cache['pairlist'] = pairlist.copy() if time_elapsed != 0.0: self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) @@ -145,6 +180,7 @@ class RemotePairList(IPairList): self.log_once(f'{info} Fetched Pairlist.', logger.info) self._last_pairlist = list(pairlist) + return pairlist def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: @@ -155,12 +191,7 @@ class RemotePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new whitelist """ - - # Validate whitelist to only have active market pairs - pairlist = self._whitelist_for_active_markets(pairlist) - pairlist = self.verify_blacklist(pairlist, logger.info) - # Limit pairlist to the requested number of pairs - pairlist = pairlist[:self._number_pairs] - self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info) - - return pairlist + rpl_pairlist = self.gen_pairlist(tickers) + merged_list = pairlist + rpl_pairlist + merged_list = sorted(set(merged_list), key=merged_list.index) + return merged_list From 6380c3d46205bdb3b29c5bff213cb7b113b93a79 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Sun, 18 Dec 2022 23:37:18 +0100 Subject: [PATCH 17/20] reduce duplicate code, fix cache check --- freqtrade/plugins/pairlist/RemotePairList.py | 54 ++++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index a0e140b42..205ee5742 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -63,6 +63,29 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + def process_json(self, jsonparse) -> Tuple[List[str], str]: + + pairlist = jsonparse.get('pairs', []) + remote_info = jsonparse.get('info', '')[:256].strip() + remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period) + + info = "".join(char if char.isalnum() or + char in " +-.,%:" else "-" for char in remote_info) + + if not self._init_done: + if self._refresh_period < remote_refresh_period: + self.log_once(f'Refresh Period has been increased from {self._refresh_period}' + f' to {remote_refresh_period} from Remote.', logger.info) + + self._refresh_period = remote_refresh_period + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + else: + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + + self._init_done = True + + return pairlist, info + def return_last_pairlist(self) -> List[str]: if self._keep_pairlist_on_failure: pairlist = self._last_pairlist @@ -91,27 +114,12 @@ class RemotePairList(IPairList): if "application/json" in str(content_type): jsonparse = response.json() - pairlist = jsonparse.get('pairs', []) - remote_info = jsonparse.get('info', '')[:256].strip() - remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period) - - info = "".join(char if char.isalnum() or - char in " +-.,%:" else "-" for char in remote_info) - - if not self._init_done and self._refresh_period < remote_refresh_period: - self.log_once(f'Refresh Period has been increased from {self._refresh_period}' - f' to {remote_refresh_period} from Remote.', logger.info) - - self._refresh_period = remote_refresh_period - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) - - self._init_done = True + pairlist, info = self.process_json(jsonparse) else: if self._init_done: self.log_once(f'Error: RemotePairList is not of type JSON: ' f' {self._pairlist_url}', logger.info) pairlist = self.return_last_pairlist() - else: raise OperationalException('RemotePairList is not of type JSON abort ') @@ -132,7 +140,7 @@ class RemotePairList(IPairList): :return: List of pairs """ - if self._init_done and self._pair_cache: + if self._init_done and self._pair_cache is not None: pairlist = self._pair_cache.get('pairlist') else: pairlist = [] @@ -142,7 +150,6 @@ class RemotePairList(IPairList): if pairlist: # Item found - no refresh necessary return pairlist.copy() - self._init_done = True else: if self._pairlist_url.startswith("file:///"): filename = self._pairlist_url.split("file:///", 1)[1] @@ -152,14 +159,7 @@ class RemotePairList(IPairList): with open(filename) as json_file: # Load the JSON data into a dictionary jsonparse = json.load(json_file) - pairlist = jsonparse['pairs'] - info = jsonparse.get('info', '') - - if not self._init_done: - self._refresh_period = jsonparse.get('refresh_period', - self._refresh_period) - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) - self._init_done = True + pairlist, info = self.process_json(jsonparse) else: raise ValueError(f"{self._pairlist_url} does not exist.") else: @@ -171,7 +171,7 @@ class RemotePairList(IPairList): pairlist = self._whitelist_for_active_markets(pairlist) pairlist = pairlist[:self._number_pairs] - if self._pair_cache: + if self._pair_cache is not None: self._pair_cache['pairlist'] = pairlist.copy() if time_elapsed != 0.0: From 43f5a16006805d763b59c3cf1f8ff32aee47df66 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Mon, 19 Dec 2022 15:36:28 +0100 Subject: [PATCH 18/20] parse exception handling, remove info, cache change --- docs/includes/pairlists.md | 3 +- freqtrade/plugins/pairlist/RemotePairList.py | 68 +++++++++++--------- tests/plugins/test_remotepairlist.py | 6 +- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 0bff9b29b..5fda038bd 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -202,11 +202,10 @@ The user is responsible for providing a server or local file that returns a JSON { "pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"], "refresh_period": 1800, - "info": "Pairlist updated on 2022-12-12 at 12:12" // Maximum Length: 256 Characters, Charset: Alphanumeric + "+-.,%:" } ``` -The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed. The `info` property is also optional and can be used to provide any additional information about the pairlist. +The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed. The optional `keep_pairlist_on_failure` specifies whether the previous received pairlist should be used if the remote server is not reachable or returns an error. The default value is true. diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 205ee5742..25530457a 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -6,7 +6,7 @@ Provides pair list fetched from a remote source import json import logging from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple import requests from cachetools import TTLCache @@ -41,7 +41,7 @@ class RemotePairList(IPairList): self._number_pairs = self._pairlistconfig['number_assets'] self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800) self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) - self._pair_cache: Optional[TTLCache] = None + self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) self._pairlist_url = self._pairlistconfig.get('pairlist_url', '') self._read_timeout = self._pairlistconfig.get('read_timeout', 60) self._bearer_token = self._pairlistconfig.get('bearer_token', '') @@ -63,28 +63,20 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." - def process_json(self, jsonparse) -> Tuple[List[str], str]: + def process_json(self, jsonparse) -> List[str]: pairlist = jsonparse.get('pairs', []) - remote_info = jsonparse.get('info', '')[:256].strip() - remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period) + remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period)) - info = "".join(char if char.isalnum() or - char in " +-.,%:" else "-" for char in remote_info) - - if not self._init_done: - if self._refresh_period < remote_refresh_period: - self.log_once(f'Refresh Period has been increased from {self._refresh_period}' - f' to {remote_refresh_period} from Remote.', logger.info) - - self._refresh_period = remote_refresh_period - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) - else: - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + if self._refresh_period < remote_refresh_period: + self.log_once(f'Refresh Period has been increased from {self._refresh_period}' + f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info) + self._refresh_period = remote_refresh_period + self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period) self._init_done = True - return pairlist, info + return pairlist def return_last_pairlist(self) -> List[str]: if self._keep_pairlist_on_failure: @@ -95,7 +87,7 @@ class RemotePairList(IPairList): return pairlist - def fetch_pairlist(self) -> Tuple[List[str], float, str]: + def fetch_pairlist(self) -> Tuple[List[str], float]: headers = { 'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist' @@ -104,8 +96,6 @@ class RemotePairList(IPairList): if self._bearer_token: headers['Authorization'] = f'Bearer {self._bearer_token}' - info = "Pairlist" - try: response = requests.get(self._pairlist_url, headers=headers, timeout=self._read_timeout) @@ -114,7 +104,17 @@ class RemotePairList(IPairList): if "application/json" in str(content_type): jsonparse = response.json() - pairlist, info = self.process_json(jsonparse) + + try: + pairlist = self.process_json(jsonparse) + except Exception as e: + + if self._init_done: + pairlist = self.return_last_pairlist() + logger.warning(f'Error while processing JSON data: {type(e)}') + else: + raise OperationalException(f'Error while processing JSON data: {type(e)}') + else: if self._init_done: self.log_once(f'Error: RemotePairList is not of type JSON: ' @@ -131,7 +131,7 @@ class RemotePairList(IPairList): time_elapsed = 0 - return pairlist, time_elapsed, info + return pairlist, time_elapsed def gen_pairlist(self, tickers: Tickers) -> List[str]: """ @@ -140,7 +140,7 @@ class RemotePairList(IPairList): :return: List of pairs """ - if self._init_done and self._pair_cache is not None: + if self._init_done: pairlist = self._pair_cache.get('pairlist') else: pairlist = [] @@ -159,25 +159,33 @@ class RemotePairList(IPairList): with open(filename) as json_file: # Load the JSON data into a dictionary jsonparse = json.load(json_file) - pairlist, info = self.process_json(jsonparse) + + try: + pairlist = self.process_json(jsonparse) + except Exception as e: + if self._init_done: + pairlist = self.return_last_pairlist() + logger.warning(f'Error while processing JSON data: {type(e)}') + else: + raise OperationalException('Error while processing' + f'JSON data: {type(e)}') else: raise ValueError(f"{self._pairlist_url} does not exist.") else: # Fetch Pairlist from Remote URL - pairlist, time_elapsed, info = self.fetch_pairlist() + pairlist, time_elapsed = self.fetch_pairlist() self.log_once(f"Fetched pairs: {pairlist}", logger.debug) pairlist = self._whitelist_for_active_markets(pairlist) pairlist = pairlist[:self._number_pairs] - if self._pair_cache is not None: - self._pair_cache['pairlist'] = pairlist.copy() + self._pair_cache['pairlist'] = pairlist.copy() if time_elapsed != 0.0: - self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) + self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info) else: - self.log_once(f'{info} Fetched Pairlist.', logger.info) + self.log_once('Fetched Pairlist.', logger.info) self._last_pairlist = list(pairlist) diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index fc91d3f06..b7a484c92 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -107,7 +107,7 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog): remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"] - pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() + pairs, time_elapsed = remote_pairlist.fetch_pairlist() assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog) assert log_has("Keeping last fetched pairlist", caplog) assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"] @@ -163,7 +163,6 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): mock_response.json.return_value = { "pairs": ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"], - "info": "Mock pairlist response", "refresh_period": 60 } @@ -179,9 +178,8 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): pairlistmanager = PairListManager(exchange, rpl_config) remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config['pairlists'][0], 0) - pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() + pairs, time_elapsed = remote_pairlist.fetch_pairlist() assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"] assert time_elapsed == 0.4 - assert info == "Mock pairlist response" assert remote_pairlist._refresh_period == 60 From ebf60d85da374a24601c8cff3ecd49fc5931fe02 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Mon, 19 Dec 2022 16:25:22 +0100 Subject: [PATCH 19/20] self._init_done placed wrong. fixed --- freqtrade/plugins/pairlist/RemotePairList.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 25530457a..0746f7e6f 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -74,7 +74,8 @@ class RemotePairList(IPairList): self._refresh_period = remote_refresh_period self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period) - self._init_done = True + + self._init_done = True return pairlist From a119fbd895f6a6ef5c33abd128b8d96f0321c520 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Dec 2022 18:19:55 +0100 Subject: [PATCH 20/20] Small error-message finetuning --- freqtrade/plugins/pairlist/RemotePairList.py | 2 +- tests/plugins/test_remotepairlist.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 0746f7e6f..b54be1fa7 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -122,7 +122,7 @@ class RemotePairList(IPairList): f' {self._pairlist_url}', logger.info) pairlist = self.return_last_pairlist() else: - raise OperationalException('RemotePairList is not of type JSON abort ') + raise OperationalException('RemotePairList is not of type JSON, abort.') except requests.exceptions.RequestException: self.log_once(f'Was not able to fetch pairlist from:' diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index b7a484c92..ac1d1f5ed 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -81,7 +81,7 @@ def test_fetch_pairlist_mock_response_html(mocker, rpl_config): remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config['pairlists'][0], 0) - with pytest.raises(OperationalException, match='RemotePairList is not of type JSON abort'): + with pytest.raises(OperationalException, match='RemotePairList is not of type JSON, abort.'): remote_pairlist.fetch_pairlist()