From 346d66748b65cf833696bc342d4c08582ad88e1a Mon Sep 17 00:00:00 2001 From: nightshift2k Date: Mon, 5 Jul 2021 12:50:56 +0200 Subject: [PATCH 1/5] first version of OffsetFilter --- freqtrade/constants.py | 6 +-- freqtrade/plugins/pairlist/OffsetFilter.py | 54 ++++++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 freqtrade/plugins/pairlist/OffsetFilter.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f4c32387b..acd143708 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -26,9 +26,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', - 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', - 'SpreadFilter', 'VolatilityFilter'] + 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', + 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', + 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py new file mode 100644 index 000000000..4579204d9 --- /dev/null +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -0,0 +1,54 @@ +""" +Offset pair list filter +""" +import logging +from typing import Any, Dict, List + +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class OffsetFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._offset = pairlistconfig.get('offset', 0) + + if self._offset < 0: + raise OperationalException("OffsetFilter requires offset to be >= 0") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Offseting pairs by {self._offset}." + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + if self._offset > len(pairlist): + self.log_once(f"Offset of {self._offset} is larger than " + + f"pair count of {len(pairlist)}", logger.warn) + pairs = pairlist[self._offset:] + self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info) + return pairs From 6cea0ef2d72e1e66bb6e496b820193b58f9d4624 Mon Sep 17 00:00:00 2001 From: nightshift2k Date: Wed, 7 Jul 2021 11:48:26 +0200 Subject: [PATCH 2/5] documentation for `OffsetFilter` --- docs/includes/pairlists.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index f19c5a181..3ae3b89ec 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) * [`AgeFilter`](#agefilter) +* [`OffsetFilter`](#offsetfilter) * [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) * [`PriceFilter`](#pricefilter) @@ -89,6 +90,30 @@ be caught out buying before the pair has finished dropping in price. This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days. +#### OffsetFilter + +Offsets an incoming pairlist by a given `offset` value. + +As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split +a larger pairlist on two bot instances. + +Example to remove the first 10 pairs from the pairlist: + +```json +"pairlists": [{ + "method": "OffsetFilter", + "offset": 10 +}], +``` + +!!! Warning + When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` + it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the + `VolumeFilter` + +!!! Note + An offset larger then the total length of the incoming pairlist will result in an empty pairlist. + #### PerformanceFilter Sorts pairs by past trade performance, as follows: From c44e87cd30a7a43a968319df326858280fffbf72 Mon Sep 17 00:00:00 2001 From: nightshift2k Date: Wed, 7 Jul 2021 12:06:55 +0200 Subject: [PATCH 3/5] added tests for `OffsetFilter to `test_pairlist.py` --- tests/plugins/test_pairlist.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index ae8f6e958..975273e60 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -417,7 +417,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "StaticPairList"}, {"method": "VolatilityFilter", "lookback_days": 3, "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}], - "BTC", ['ETH/BTC', 'TKN/BTC']) + "BTC", ['ETH/BTC', 'TKN/BTC']), + # VolumePairList with no offset = unchanged pairlist + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 0}], + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with offset = 2 + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 2}], + "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with higher offset, than total pairlist + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 100}], + "USDT", []) ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, pairlists, base_currency, @@ -695,6 +707,18 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 +def test_OffsetFilter_error(mocker, whitelist_conf) -> None: + whitelist_conf['pairlists'] = ( + [{"method": "StaticPairList"}, {"method": "OffsetFilter", "offset": -1}] + ) + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + with pytest.raises(OperationalException, + match=r'OffsetFilter requires offset to be >= 0'): + PairListManager(MagicMock, whitelist_conf) + + def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'RangeStabilityFilter', 'lookback_days': 99999}] From 4d4ed82db8d41904d45302df2999bc660f8daed3 Mon Sep 17 00:00:00 2001 From: nightshift2k <82537832+nightshift2k@users.noreply.github.com> Date: Wed, 7 Jul 2021 20:29:52 +0200 Subject: [PATCH 4/5] Update docs/includes/pairlists.md Co-authored-by: Matthias --- docs/includes/pairlists.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 3ae3b89ec..5b2e892e8 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -109,7 +109,7 @@ Example to remove the first 10 pairs from the pairlist: !!! Warning When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the - `VolumeFilter` + `VolumeFilter`. !!! Note An offset larger then the total length of the incoming pairlist will result in an empty pairlist. From 5a2bc192d4728f7c26e2ded30a68a833df0aa6c1 Mon Sep 17 00:00:00 2001 From: nightshift2k <82537832+nightshift2k@users.noreply.github.com> Date: Wed, 7 Jul 2021 20:29:55 +0200 Subject: [PATCH 5/5] Update docs/includes/pairlists.md Co-authored-by: Matthias --- docs/includes/pairlists.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 5b2e892e8..ecb6231ec 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -100,10 +100,12 @@ a larger pairlist on two bot instances. Example to remove the first 10 pairs from the pairlist: ```json -"pairlists": [{ +"pairlists": [ + { "method": "OffsetFilter", "offset": 10 -}], + } +], ``` !!! Warning