From a323acf343ba1f5622cd9e47ea13905eb97df4f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Oct 2022 09:46:12 +0100 Subject: [PATCH 1/2] Improve ShufflePairlist to shuffle only once per candle --- docs/includes/pairlists.md | 12 ++++++++++++ freqtrade/plugins/pairlist/IPairList.py | 1 - freqtrade/plugins/pairlist/ShuffleFilter.py | 17 +++++++++++++++-- tests/plugins/test_pairlist.py | 21 +++++++++++++++++---- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 7dff75a02..5c1bf8274 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -286,6 +286,18 @@ Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. +By default, ShuffleFilter will shuffle pairs once per candle. +To shuffle on every iteration, set `"shuffle"` to `"iteration"` instead of the default of `"candle"`. + +``` json + { + "method": "ShuffleFilter", + "shuffle": "candle", + "seed": 42 + } + +``` + !!! Tip You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. ShuffleFilter will automatically detect runmodes and apply the `seed` only for backtesting modes - if a `seed` value is set. diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 660d6228c..d0382c778 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -36,7 +36,6 @@ class IPairList(LoggingMixin, ABC): self._pairlistconfig = pairlistconfig self._pairlist_pos = pairlist_pos self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) - self._last_refresh = 0 LoggingMixin.__init__(self, logger, self.refresh_period) @property diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 1bc114d4e..e67ef52e8 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -3,16 +3,20 @@ Shuffle pair list filter """ import logging import random -from typing import Any, Dict, List +from typing import Any, Dict, List, Literal from freqtrade.constants import Config from freqtrade.enums import RunMode +from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util.periodic_cache import PeriodicCache logger = logging.getLogger(__name__) +ShuffleValues = Literal['candle', 'iteration'] + class ShuffleFilter(IPairList): @@ -31,6 +35,9 @@ class ShuffleFilter(IPairList): logger.info(f"Backtesting mode detected, applying seed value: {self._seed}") self._random = random.Random(self._seed) + self._shuffle: ShuffleValues = pairlistconfig.get('shuffle', 'candle') + self.__pairlist_cache = PeriodicCache( + maxsize=1000, ttl=timeframe_to_seconds(self._config['timeframe'])) @property def needstickers(self) -> bool: @@ -45,7 +52,7 @@ class ShuffleFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return (f"{self.name} - Shuffling pairs" + + return (f"{self.name} - Shuffling pairs every {self._shuffle}" + (f", seed = {self._seed}." if self._seed is not None else ".")) def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: @@ -56,7 +63,13 @@ class ShuffleFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new whitelist """ + pairlist_bef = tuple(pairlist) + pairlist_new = self.__pairlist_cache.get(pairlist_bef) + if pairlist_new and self._shuffle == 'candle': + # Use cached pairlist. + return pairlist_new # Shuffle is done inplace self._random.shuffle(pairlist) + self.__pairlist_cache[pairlist_bef] = pairlist return pairlist diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index f0b983063..359291476 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -2,6 +2,8 @@ import logging import time +from copy import deepcopy +from datetime import timedelta from unittest.mock import MagicMock, PropertyMock import pandas as pd @@ -719,15 +721,26 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: whitelist_conf['pairlists'] = [ {"method": "StaticPairList"}, - {"method": "ShuffleFilter", "seed": 42} + {"method": "ShuffleFilter", "seed": 43} ] exchange = get_patched_exchange(mocker, whitelist_conf) - PairListManager(exchange, whitelist_conf) - assert log_has("Backtesting mode detected, applying seed value: 42", caplog) + plm = PairListManager(exchange, whitelist_conf) + assert log_has("Backtesting mode detected, applying seed value: 43", caplog) + + with time_machine.travel("2021-09-01 05:01:00 +00:00") as t: + plm.refresh_pairlist() + pl1 = deepcopy(plm.whitelist) + plm.refresh_pairlist() + assert plm.whitelist == pl1 + + t.shift(timedelta(minutes=10)) + plm.refresh_pairlist() + assert plm.whitelist != pl1 + caplog.clear() whitelist_conf['runmode'] = RunMode.DRY_RUN - PairListManager(exchange, whitelist_conf) + plm = PairListManager(exchange, whitelist_conf) assert not log_has("Backtesting mode detected, applying seed value: 42", caplog) assert log_has("Live mode detected, not applying seed.", caplog) From 5013351143489d3eb48507340a651eadf6b5abed Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Oct 2022 09:48:55 +0100 Subject: [PATCH 2/2] Rename "shuffle" parameter to "shuffle_freq" --- docs/includes/pairlists.md | 4 ++-- freqtrade/plugins/pairlist/ShuffleFilter.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 5c1bf8274..dd3149e98 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -287,12 +287,12 @@ Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. By default, ShuffleFilter will shuffle pairs once per candle. -To shuffle on every iteration, set `"shuffle"` to `"iteration"` instead of the default of `"candle"`. +To shuffle on every iteration, set `"shuffle_frequency"` to `"iteration"` instead of the default of `"candle"`. ``` json { "method": "ShuffleFilter", - "shuffle": "candle", + "shuffle_frequency": "candle", "seed": 42 } diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index e67ef52e8..76d7600d2 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -35,7 +35,7 @@ class ShuffleFilter(IPairList): logger.info(f"Backtesting mode detected, applying seed value: {self._seed}") self._random = random.Random(self._seed) - self._shuffle: ShuffleValues = pairlistconfig.get('shuffle', 'candle') + self._shuffle_freq: ShuffleValues = pairlistconfig.get('shuffle_frequency', 'candle') self.__pairlist_cache = PeriodicCache( maxsize=1000, ttl=timeframe_to_seconds(self._config['timeframe'])) @@ -52,7 +52,7 @@ class ShuffleFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return (f"{self.name} - Shuffling pairs every {self._shuffle}" + + return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" + (f", seed = {self._seed}." if self._seed is not None else ".")) def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: @@ -65,7 +65,7 @@ class ShuffleFilter(IPairList): """ pairlist_bef = tuple(pairlist) pairlist_new = self.__pairlist_cache.get(pairlist_bef) - if pairlist_new and self._shuffle == 'candle': + if pairlist_new and self._shuffle_freq == 'candle': # Use cached pairlist. return pairlist_new # Shuffle is done inplace