Merge pull request #3168 from freqtrade/fix_pairlist_caching

Fix pairlist caching
This commit is contained in:
hroff-1902 2020-04-16 18:39:00 +03:00 committed by GitHub
commit 9364a9c4c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 63 additions and 18 deletions

View File

@ -9,6 +9,8 @@ from abc import ABC, abstractmethod, abstractproperty
from copy import deepcopy from copy import deepcopy
from typing import Any, Dict, List from typing import Any, Dict, List
from cachetools import TTLCache, cached
from freqtrade.exchange import market_is_active from freqtrade.exchange import market_is_active
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -31,6 +33,9 @@ class IPairList(ABC):
self._config = config self._config = config
self._pairlistconfig = pairlistconfig self._pairlistconfig = pairlistconfig
self._pairlist_pos = pairlist_pos self._pairlist_pos = pairlist_pos
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._last_refresh = 0
self._log_cache = TTLCache(maxsize=1024, ttl=self.refresh_period)
@property @property
def name(self) -> str: def name(self) -> str:
@ -40,6 +45,24 @@ class IPairList(ABC):
""" """
return self.__class__.__name__ return self.__class__.__name__
def log_on_refresh(self, logmethod, message: str) -> None:
"""
Logs message - not more often than "refresh_period" to avoid log spamming
Logs the log-message as debug as well to simplify debugging.
:param logmethod: Function that'll be called. Most likely `logger.info`.
:param message: String containing the message to be sent to the function.
:return: None.
"""
@cached(cache=self._log_cache)
def _log_on_refresh(message: str):
logmethod(message)
# Log as debug first
logger.debug(message)
# Call hidden function.
_log_on_refresh(message)
@abstractproperty @abstractproperty
def needstickers(self) -> bool: def needstickers(self) -> bool:
""" """

View File

@ -39,7 +39,8 @@ class PrecisionFilter(IPairList):
stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99) stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99)
logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
if sp <= stop_gap_price: if sp <= stop_gap_price:
logger.info(f"Removed {ticker['symbol']} from whitelist, " self.log_on_refresh(logger.info,
f"Removed {ticker['symbol']} from whitelist, "
f"because stop price {sp} would be <= stop limit {stop_gap_price}") f"because stop price {sp} would be <= stop limit {stop_gap_price}")
return False return False
return True return True

View File

@ -41,7 +41,7 @@ class PriceFilter(IPairList):
ticker['last']) ticker['last'])
changeperc = (compare - ticker['last']) / ticker['last'] changeperc = (compare - ticker['last']) / ticker['last']
if changeperc > self._low_price_ratio: if changeperc > self._low_price_ratio:
logger.info(f"Removed {ticker['symbol']} from whitelist, " self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because 1 unit is {changeperc * 100:.3f}%") f"because 1 unit is {changeperc * 100:.3f}%")
return False return False
return True return True

View File

@ -49,7 +49,7 @@ class SpreadFilter(IPairList):
if 'bid' in ticker and 'ask' in ticker: if 'bid' in ticker and 'ask' in ticker:
spread = 1 - ticker['bid'] / ticker['ask'] spread = 1 - ticker['bid'] / ticker['ask']
if not ticker or spread > self._max_spread_ratio: if not ticker or spread > self._max_spread_ratio:
logger.info(f"Removed {ticker['symbol']} from whitelist, " self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because spread {spread * 100:.3f}% >" f"because spread {spread * 100:.3f}% >"
f"{self._max_spread_ratio * 100}%") f"{self._max_spread_ratio * 100}%")
pairlist.remove(p) pairlist.remove(p)

View File

@ -39,7 +39,6 @@ class VolumePairList(IPairList):
if not self._validate_keys(self._sort_key): if not self._validate_keys(self._sort_key):
raise OperationalException( raise OperationalException(
f'key {self._sort_key} not in {SORT_VALUES}') f'key {self._sort_key} not in {SORT_VALUES}')
self._last_refresh = 0
@property @property
def needstickers(self) -> bool: def needstickers(self) -> bool:
@ -68,16 +67,18 @@ class VolumePairList(IPairList):
:return: new whitelist :return: new whitelist
""" """
# Generate dynamic whitelist # Generate dynamic whitelist
if self._last_refresh + self.refresh_period < datetime.now().timestamp(): # Must always run if this pairlist is not the first in the list.
if (self._pairlist_pos != 0 or
(self._last_refresh + self.refresh_period < datetime.now().timestamp())):
self._last_refresh = int(datetime.now().timestamp()) self._last_refresh = int(datetime.now().timestamp())
return self._gen_pair_whitelist(pairlist, pairs = self._gen_pair_whitelist(pairlist, tickers,
tickers,
self._config['stake_currency'], self._config['stake_currency'],
self._sort_key, self._sort_key, self._min_value)
self._min_value
)
else: else:
return pairlist pairs = pairlist
self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
return pairs
def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict, def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict,
base_currency: str, key: str, min_val: int) -> List[str]: base_currency: str, key: str, min_val: int) -> List[str]:
@ -88,7 +89,6 @@ class VolumePairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). :param tickers: Tickers (from exchange.get_tickers()).
:return: List of pairs :return: List of pairs
""" """
if self._pairlist_pos == 0: if self._pairlist_pos == 0:
# If VolumePairList is the first in the list, use fresh pairlist # If VolumePairList is the first in the list, use fresh pairlist
# Check if pair quote currency equals to the stake currency. # Check if pair quote currency equals to the stake currency.
@ -109,6 +109,5 @@ class VolumePairList(IPairList):
pairs = self._verify_blacklist(pairs, aswarning=False) pairs = self._verify_blacklist(pairs, aswarning=False)
# Limit to X number of pairs # Limit to X number of pairs
pairs = pairs[:self._number_pairs] pairs = pairs[:self._number_pairs]
logger.info(f"Searching {self._number_pairs} pairs: {pairs}")
return pairs return pairs

View File

@ -46,6 +46,28 @@ def static_pl_conf(whitelist_conf):
return whitelist_conf return whitelist_conf
def test_log_on_refresh(mocker, static_pl_conf, markets, tickers):
mocker.patch.multiple('freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets),
exchange_has=MagicMock(return_value=True),
get_tickers=tickers
)
freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
logmock = MagicMock()
# Assign starting whitelist
pl = freqtrade.pairlists._pairlists[0]
pl.log_on_refresh(logmock, 'Hello world')
assert logmock.call_count == 1
pl.log_on_refresh(logmock, 'Hello world')
assert logmock.call_count == 1
assert pl._log_cache.currsize == 1
assert ('Hello world',) in pl._log_cache._Cache__data
pl.log_on_refresh(logmock, 'Hello world2')
assert logmock.call_count == 2
assert pl._log_cache.currsize == 2
def test_load_pairlist_noexist(mocker, markets, default_conf): def test_load_pairlist_noexist(mocker, markets, default_conf):
bot = get_patched_freqtradebot(mocker, default_conf) bot = get_patched_freqtradebot(mocker, default_conf)
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))