From 3cdd06f5627b7baee7b5ec3ead09e036080e9b66 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Sep 2021 19:32:51 +0200 Subject: [PATCH 1/5] Add PeriodicCache --- freqtrade/configuration/PeriodicCache.py | 19 +++++++++++++++ freqtrade/configuration/__init__.py | 1 + requirements-dev.txt | 2 ++ tests/test_periodiccache.py | 31 ++++++++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100644 freqtrade/configuration/PeriodicCache.py create mode 100644 tests/test_periodiccache.py diff --git a/freqtrade/configuration/PeriodicCache.py b/freqtrade/configuration/PeriodicCache.py new file mode 100644 index 000000000..25c0c47f3 --- /dev/null +++ b/freqtrade/configuration/PeriodicCache.py @@ -0,0 +1,19 @@ +from datetime import datetime, timezone + +from cachetools.ttl import TTLCache + + +class PeriodicCache(TTLCache): + """ + Special cache that expires at "straight" times + A timer with ttl of 3600 (1h) will expire at every full hour (:00). + """ + + def __init__(self, maxsize, ttl, getsizeof=None): + def local_timer(): + ts = datetime.now(timezone.utc).timestamp() + offset = (ts % ttl) + return ts - offset + + # Init with smlight offset + super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 607f9cdef..b1b268092 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency diff --git a/requirements-dev.txt b/requirements-dev.txt index 34d5607f3..4859e1cc6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,6 +14,8 @@ pytest-cov==2.12.1 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.9.3 +# For datetime mocking +time-machine==2.4.0 # Convert jupyter notebooks to markdown documents nbconvert==6.1.0 diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py new file mode 100644 index 000000000..ff9b53684 --- /dev/null +++ b/tests/test_periodiccache.py @@ -0,0 +1,31 @@ +from freqtrade.configuration import PeriodicCache +import time_machine + + +def test_ttl_cache(): + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + + cache = PeriodicCache(5, ttl=60) + cache1h = PeriodicCache(5, ttl=3600) + + assert cache.timer() == 1630472400.0 + cache['a'] = 1235 + cache1h['a'] = 555123 + assert 'a' in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:00:59 +00:00") + assert 'a' in cache + assert 'a' in cache1h + + # Cache expired + t.move_to("2021-09-01 05:01:00 +00:00") + assert 'a' not in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:59:59 +00:00") + assert 'a' in cache1h + + t.move_to("2021-09-01 06:00:00 +00:00") + assert 'a' not in cache1h From 8afb3c4b70c3458772953c8be23976bd90a264ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Sep 2021 19:33:28 +0200 Subject: [PATCH 2/5] Move AgeFilter cache to instance level --- freqtrade/plugins/pairlist/AgeFilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index dc5cab31e..32fef5fb0 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -18,14 +18,14 @@ logger = logging.getLogger(__name__) class AgeFilter(IPairList): - # Checked symbols cache (dictionary of ticker symbol => timestamp) - _symbolsChecked: Dict[str, int] = {} - def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + # Checked symbols cache (dictionary of ticker symbol => timestamp) + self._symbolsChecked: Dict[str, int] = {} + self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) From c9ba52d7321b2cae17776c66723c837a75c3bfdf Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 06:30:18 +0200 Subject: [PATCH 3/5] Expire cached pairs in age-filter once per day --- freqtrade/plugins/pairlist/AgeFilter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 32fef5fb0..1fba00123 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -11,6 +11,7 @@ from pandas import DataFrame from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.configuration import PeriodicCache logger = logging.getLogger(__name__) @@ -25,6 +26,7 @@ class AgeFilter(IPairList): # Checked symbols cache (dictionary of ticker symbol => timestamp) self._symbolsChecked: Dict[str, int] = {} + self._too_young_pairs = PeriodicCache(maxsize=1000, ttl=86_400) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) @@ -69,10 +71,12 @@ class AgeFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] + needed_pairs = [ + (p, '1d') for p in pairlist + if p not in self._symbolsChecked and p not in self._too_young_pairs] if not needed_pairs: return pairlist - + logger.info(f"needed pairs {needed_pairs}") since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed ) - 1 @@ -118,5 +122,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) + self._too_young_pairs[pair] = arrow.utcnow().int_timestamp * 1000 return False return False From 3ce5197e8d07471e75be14df435c3d291b3b69c4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 06:45:26 +0200 Subject: [PATCH 4/5] Add Tests for AgeFilter caching closes #5552 --- freqtrade/configuration/__init__.py | 2 +- freqtrade/plugins/pairlist/AgeFilter.py | 11 ++-- tests/plugins/test_pairlist.py | 69 ++++++++++++++++--------- tests/test_periodiccache.py | 3 +- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index b1b268092..dccbb14b3 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,8 +1,8 @@ # flake8: noqa: F401 -from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.timerange import TimeRange diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 1fba00123..c43bd0c4c 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -8,10 +8,10 @@ from typing import Any, Dict, List, Optional import arrow from pandas import DataFrame +from freqtrade.configuration import PeriodicCache from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList -from freqtrade.configuration import PeriodicCache logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ class AgeFilter(IPairList): # Checked symbols cache (dictionary of ticker symbol => timestamp) self._symbolsChecked: Dict[str, int] = {} - self._too_young_pairs = PeriodicCache(maxsize=1000, ttl=86_400) + self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) @@ -73,9 +73,10 @@ class AgeFilter(IPairList): """ needed_pairs = [ (p, '1d') for p in pairlist - if p not in self._symbolsChecked and p not in self._too_young_pairs] + if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: - return pairlist + # Remove pairs that have been removed before + return [p for p in pairlist if p not in self._symbolsCheckFailed] logger.info(f"needed pairs {needed_pairs}") since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed @@ -122,6 +123,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) - self._too_young_pairs[pair] = arrow.utcnow().int_timestamp * 1000 + self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000 return False return False diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5f0701a22..3cdf6d341 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -4,6 +4,7 @@ import time from unittest.mock import MagicMock, PropertyMock import pytest +import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException @@ -815,32 +816,52 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): - ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, - } - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), - exchange_has=MagicMock(return_value=True), - get_tickers=tickers - ) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), - ) + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], + } + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers, + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) - freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - # Called once for XRP/BTC - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + # Call to XRP/BTC cached + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + # Move to next day + t.move_to("2021-09-02 01:00:00 +00:00") + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + + # Move another day with fresh mocks (now the pair is old enough) + t.move_to("2021-09-03 01:00:00 +00:00") + # Called once for XRP/BTC + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 4 + # Called once (only for XRP/BTC) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 def test_OffsetFilter_error(mocker, whitelist_conf) -> None: diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py index ff9b53684..f874f9041 100644 --- a/tests/test_periodiccache.py +++ b/tests/test_periodiccache.py @@ -1,6 +1,7 @@ -from freqtrade.configuration import PeriodicCache import time_machine +from freqtrade.configuration import PeriodicCache + def test_ttl_cache(): From 35eda8c8c7c304051cae6b2a22453f56aa0ea4a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 07:07:20 +0200 Subject: [PATCH 5/5] Improve agefilter test --- tests/plugins/test_pairlist.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 3cdf6d341..34770c03d 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -821,7 +821,6 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o ('ETH/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history, ('LTC/BTC', '1d'): ohlcv_history, - ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], } mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -837,16 +836,28 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert len(freqtrade.pairlists.whitelist) == 3 assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3 # Call to XRP/BTC cached - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count - # Move to next day - t.move_to("2021-09-02 01:00:00 +00:00") + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2 + + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3 - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 + + # Move to next day + t.move_to("2021-09-02 01:00:00 +00:00") + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 # Move another day with fresh mocks (now the pair is old enough) t.move_to("2021-09-03 01:00:00 +00:00")