diff --git a/freqtrade/configuration/PeriodicCache.py b/freqtrade/configuration/PeriodicCache.py new file mode 100644 index 000000000..25c0c47f3 --- /dev/null +++ b/freqtrade/configuration/PeriodicCache.py @@ -0,0 +1,19 @@ +from datetime import datetime, timezone + +from cachetools.ttl import TTLCache + + +class PeriodicCache(TTLCache): + """ + Special cache that expires at "straight" times + A timer with ttl of 3600 (1h) will expire at every full hour (:00). + """ + + def __init__(self, maxsize, ttl, getsizeof=None): + def local_timer(): + ts = datetime.now(timezone.utc).timestamp() + offset = (ts % ttl) + return ts - offset + + # Init with smlight offset + super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 730a4e47f..cf41c0ca9 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -4,4 +4,5 @@ from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.timerange import TimeRange diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index dc5cab31e..c43bd0c4c 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional import arrow from pandas import DataFrame +from freqtrade.configuration import PeriodicCache from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -18,14 +19,15 @@ logger = logging.getLogger(__name__) class AgeFilter(IPairList): - # Checked symbols cache (dictionary of ticker symbol => timestamp) - _symbolsChecked: Dict[str, int] = {} - def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + # Checked symbols cache (dictionary of ticker symbol => timestamp) + self._symbolsChecked: Dict[str, int] = {} + self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400) + self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) @@ -69,10 +71,13 @@ class AgeFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] + needed_pairs = [ + (p, '1d') for p in pairlist + if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: - return pairlist - + # Remove pairs that have been removed before + return [p for p in pairlist if p not in self._symbolsCheckFailed] + logger.info(f"needed pairs {needed_pairs}") since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed ) - 1 @@ -118,5 +123,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) + self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000 return False return False diff --git a/requirements-dev.txt b/requirements-dev.txt index 34d5607f3..4859e1cc6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,6 +14,8 @@ pytest-cov==2.12.1 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.9.3 +# For datetime mocking +time-machine==2.4.0 # Convert jupyter notebooks to markdown documents nbconvert==6.1.0 diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5f0701a22..34770c03d 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -4,6 +4,7 @@ import time from unittest.mock import MagicMock, PropertyMock import pytest +import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException @@ -815,32 +816,63 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): - ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, - } - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), - exchange_has=MagicMock(return_value=True), - get_tickers=tickers - ) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), - ) + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + } + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers, + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) - freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - # Called once for XRP/BTC - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + # Call to XRP/BTC cached + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2 + + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 + + # Move to next day + t.move_to("2021-09-02 01:00:00 +00:00") + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 + + # Move another day with fresh mocks (now the pair is old enough) + t.move_to("2021-09-03 01:00:00 +00:00") + # Called once for XRP/BTC + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 4 + # Called once (only for XRP/BTC) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 def test_OffsetFilter_error(mocker, whitelist_conf) -> None: diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py new file mode 100644 index 000000000..f874f9041 --- /dev/null +++ b/tests/test_periodiccache.py @@ -0,0 +1,32 @@ +import time_machine + +from freqtrade.configuration import PeriodicCache + + +def test_ttl_cache(): + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + + cache = PeriodicCache(5, ttl=60) + cache1h = PeriodicCache(5, ttl=3600) + + assert cache.timer() == 1630472400.0 + cache['a'] = 1235 + cache1h['a'] = 555123 + assert 'a' in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:00:59 +00:00") + assert 'a' in cache + assert 'a' in cache1h + + # Cache expired + t.move_to("2021-09-01 05:01:00 +00:00") + assert 'a' not in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:59:59 +00:00") + assert 'a' in cache1h + + t.move_to("2021-09-01 06:00:00 +00:00") + assert 'a' not in cache1h