Merge pull request #5574 from freqtrade/agefilter_cache

Agefilter cache
This commit is contained in:
Matthias 2021-09-15 06:33:36 +02:00 committed by GitHub
commit e4ec5679a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 122 additions and 30 deletions

View File

@ -0,0 +1,19 @@
from datetime import datetime, timezone
from cachetools.ttl import TTLCache
class PeriodicCache(TTLCache):
"""
Special cache that expires at "straight" times
A timer with ttl of 3600 (1h) will expire at every full hour (:00).
"""
def __init__(self, maxsize, ttl, getsizeof=None):
def local_timer():
ts = datetime.now(timezone.utc).timestamp()
offset = (ts % ttl)
return ts - offset
# Init with smlight offset
super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)

View File

@ -4,4 +4,5 @@ from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.PeriodicCache import PeriodicCache
from freqtrade.configuration.timerange import TimeRange from freqtrade.configuration.timerange import TimeRange

View File

@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
import arrow import arrow
from pandas import DataFrame from pandas import DataFrame
from freqtrade.configuration import PeriodicCache
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.plugins.pairlist.IPairList import IPairList
@ -18,14 +19,15 @@ logger = logging.getLogger(__name__)
class AgeFilter(IPairList): class AgeFilter(IPairList):
# Checked symbols cache (dictionary of ticker symbol => timestamp)
_symbolsChecked: Dict[str, int] = {}
def __init__(self, exchange, pairlistmanager, def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any], config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None: pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
# Checked symbols cache (dictionary of ticker symbol => timestamp)
self._symbolsChecked: Dict[str, int] = {}
self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
self._max_days_listed = pairlistconfig.get('max_days_listed', None) self._max_days_listed = pairlistconfig.get('max_days_listed', None)
@ -69,10 +71,13 @@ class AgeFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached. :param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist :return: new allowlist
""" """
needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] needed_pairs = [
(p, '1d') for p in pairlist
if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
if not needed_pairs: if not needed_pairs:
return pairlist # Remove pairs that have been removed before
return [p for p in pairlist if p not in self._symbolsCheckFailed]
logger.info(f"needed pairs {needed_pairs}")
since_days = -( since_days = -(
self._max_days_listed if self._max_days_listed else self._min_days_listed self._max_days_listed if self._max_days_listed else self._min_days_listed
) - 1 ) - 1
@ -118,5 +123,6 @@ class AgeFilter(IPairList):
" or more than " " or more than "
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
) if self._max_days_listed else ''), logger.info) ) if self._max_days_listed else ''), logger.info)
self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
return False return False
return False return False

View File

@ -14,6 +14,8 @@ pytest-cov==2.12.1
pytest-mock==3.6.1 pytest-mock==3.6.1
pytest-random-order==1.0.4 pytest-random-order==1.0.4
isort==5.9.3 isort==5.9.3
# For datetime mocking
time-machine==2.4.0
# Convert jupyter notebooks to markdown documents # Convert jupyter notebooks to markdown documents
nbconvert==6.1.0 nbconvert==6.1.0

View File

@ -4,6 +4,7 @@ import time
from unittest.mock import MagicMock, PropertyMock from unittest.mock import MagicMock, PropertyMock
import pytest import pytest
import time_machine
from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.constants import AVAILABLE_PAIRLISTS
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
@ -815,32 +816,63 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
ohlcv_data = { with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
('ETH/BTC', '1d'): ohlcv_history, ohlcv_data = {
('TKN/BTC', '1d'): ohlcv_history, ('ETH/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history,
} ('LTC/BTC', '1d'): ohlcv_history,
mocker.patch.multiple('freqtrade.exchange.Exchange', }
markets=PropertyMock(return_value=markets), mocker.patch.multiple(
exchange_has=MagicMock(return_value=True), 'freqtrade.exchange.Exchange',
get_tickers=tickers markets=PropertyMock(return_value=markets),
) exchange_has=MagicMock(return_value=True),
mocker.patch.multiple( get_tickers=tickers,
'freqtrade.exchange.Exchange', refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), )
)
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
freqtrade.pairlists.refresh_pairlist() freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3 assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist()
freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3
assert len(freqtrade.pairlists.whitelist) == 3 # Call to XRP/BTC cached
# Called once for XRP/BTC assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
}
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
# Move to next day
t.move_to("2021-09-02 01:00:00 +00:00")
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
# Move another day with fresh mocks (now the pair is old enough)
t.move_to("2021-09-03 01:00:00 +00:00")
# Called once for XRP/BTC
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
('XRP/BTC', '1d'): ohlcv_history,
}
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 4
# Called once (only for XRP/BTC)
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
def test_OffsetFilter_error(mocker, whitelist_conf) -> None: def test_OffsetFilter_error(mocker, whitelist_conf) -> None:

View File

@ -0,0 +1,32 @@
import time_machine
from freqtrade.configuration import PeriodicCache
def test_ttl_cache():
with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
cache = PeriodicCache(5, ttl=60)
cache1h = PeriodicCache(5, ttl=3600)
assert cache.timer() == 1630472400.0
cache['a'] = 1235
cache1h['a'] = 555123
assert 'a' in cache
assert 'a' in cache1h
t.move_to("2021-09-01 05:00:59 +00:00")
assert 'a' in cache
assert 'a' in cache1h
# Cache expired
t.move_to("2021-09-01 05:01:00 +00:00")
assert 'a' not in cache
assert 'a' in cache1h
t.move_to("2021-09-01 05:59:59 +00:00")
assert 'a' in cache1h
t.move_to("2021-09-01 06:00:00 +00:00")
assert 'a' not in cache1h