Add Tests for AgeFilter caching

closes #5552

freqtrade/configuration/__init__.py

@@ -1,8 +1,8 @@
 # flake8: noqa: F401
 
-from freqtrade.configuration.PeriodicCache import PeriodicCache
 from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration
+from freqtrade.configuration.PeriodicCache import PeriodicCache
 from freqtrade.configuration.timerange import TimeRange

freqtrade/plugins/pairlist/AgeFilter.py

@@ -8,10 +8,10 @@ from typing import Any, Dict, List, Optional
 import arrow
 from pandas import DataFrame
 
+from freqtrade.configuration import PeriodicCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import plural
 from freqtrade.plugins.pairlist.IPairList import IPairList
-from freqtrade.configuration import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ class AgeFilter(IPairList):
 
         # Checked symbols cache (dictionary of ticker symbol => timestamp)
         self._symbolsChecked: Dict[str, int] = {}
-        self._too_young_pairs = PeriodicCache(maxsize=1000, ttl=86_400)
+        self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
 
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
         self._max_days_listed = pairlistconfig.get('max_days_listed', None)
@@ -73,9 +73,10 @@ class AgeFilter(IPairList):
         """
         needed_pairs = [
             (p, '1d') for p in pairlist
-            if p not in self._symbolsChecked and p not in self._too_young_pairs]
+            if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
         if not needed_pairs:
-            return pairlist
+            # Remove pairs that have been removed before
+            return [p for p in pairlist if p not in self._symbolsCheckFailed]
         logger.info(f"needed pairs {needed_pairs}")
         since_days = -(
             self._max_days_listed if self._max_days_listed else self._min_days_listed
@@ -122,6 +123,6 @@ class AgeFilter(IPairList):
                     " or more than "
                     f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
                 ) if self._max_days_listed else ''), logger.info)
-                self._too_young_pairs[pair] = arrow.utcnow().int_timestamp * 1000
+                self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
                 return False
         return False

tests/plugins/test_pairlist.py

@@ -4,6 +4,7 @@ import time
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
+import time_machine
 
 from freqtrade.constants import AVAILABLE_PAIRLISTS
 from freqtrade.exceptions import OperationalException
@@ -815,18 +816,18 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
 
 
 def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
+    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
         ohlcv_data = {
             ('ETH/BTC', '1d'): ohlcv_history,
             ('TKN/BTC', '1d'): ohlcv_history,
             ('LTC/BTC', '1d'): ohlcv_history,
+            ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
         }
-    mocker.patch.multiple('freqtrade.exchange.Exchange',
-                          markets=PropertyMock(return_value=markets),
-                          exchange_has=MagicMock(return_value=True),
-                          get_tickers=tickers
-                          )
         mocker.patch.multiple(
             'freqtrade.exchange.Exchange',
+            markets=PropertyMock(return_value=markets),
+            exchange_has=MagicMock(return_value=True),
+            get_tickers=tickers,
             refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
         )
 
@@ -839,9 +840,29 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o
         previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
         freqtrade.pairlists.refresh_pairlist()
         assert len(freqtrade.pairlists.whitelist) == 3
-    # Called once for XRP/BTC
+        # Call to XRP/BTC cached
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count
+        # Move to next day
+        t.move_to("2021-09-02 01:00:00 +00:00")
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
         assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
 
+        # Move another day with fresh mocks (now the pair is old enough)
+        t.move_to("2021-09-03 01:00:00 +00:00")
+        # Called once for XRP/BTC
+        ohlcv_data = {
+            ('ETH/BTC', '1d'): ohlcv_history,
+            ('TKN/BTC', '1d'): ohlcv_history,
+            ('LTC/BTC', '1d'): ohlcv_history,
+            ('XRP/BTC', '1d'): ohlcv_history,
+        }
+        mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 4
+        # Called once (only for XRP/BTC)
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
+
 
 def test_OffsetFilter_error(mocker, whitelist_conf) -> None:
     whitelist_conf['pairlists'] = (

tests/test_periodiccache.py

@@ -1,6 +1,7 @@
-from freqtrade.configuration import PeriodicCache
 import time_machine
 
+from freqtrade.configuration import PeriodicCache
+
 
 def test_ttl_cache():