Merge pull request #5246 from nightshift2k/feature/offsetfilter

new filter OffsetFilter for offsetting incoming pairlists

docs/includes/pairlists.md

@@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`StaticPairList`](#static-pair-list) (default, if not configured differently)
 * [`VolumePairList`](#volume-pair-list)
 * [`AgeFilter`](#agefilter)
+* [`OffsetFilter`](#offsetfilter)
 * [`PerformanceFilter`](#performancefilter)
 * [`PrecisionFilter`](#precisionfilter)
 * [`PriceFilter`](#pricefilter)
@@ -89,6 +90,32 @@ be caught out buying before the pair has finished dropping in price.
 
 This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`.
 
+#### OffsetFilter
+
+Offsets an incoming pairlist by a given `offset` value.
+
+As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split
+a larger pairlist on two bot instances.
+
+Example to remove the first 10 pairs from the pairlist:
+
+```json
+"pairlists": [
+    {
+        "method": "OffsetFilter",
+        "offset": 10
+    }
+],
+```
+
+!!! Warning
+    When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` 
+    it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the
+    `VolumeFilter`.
+
+!!! Note
+    An offset larger then the total length of the incoming pairlist will result in an empty pairlist.
+
 #### PerformanceFilter
 
 Sorts pairs by past trade performance, as follows:

freqtrade/constants.py

@@ -26,9 +26,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
                          'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
-                       'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
-                       'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
-                       'SpreadFilter', 'VolatilityFilter']
+                       'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
+                       'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
+                       'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000

freqtrade/plugins/pairlist/OffsetFilter.py

@@ -0,0 +1,54 @@
+"""
+Offset pair list filter
+"""
+import logging
+from typing import Any, Dict, List
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class OffsetFilter(IPairList):
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._offset = pairlistconfig.get('offset', 0)
+
+        if self._offset < 0:
+            raise OperationalException("OffsetFilter requires offset to be >= 0")
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty Dict is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - Offseting pairs by {self._offset}."
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
+        """
+        if self._offset > len(pairlist):
+            self.log_once(f"Offset of {self._offset} is larger than " +
+                          f"pair count of {len(pairlist)}", logger.warn)
+        pairs = pairlist[self._offset:]
+        self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info)
+        return pairs

tests/plugins/test_pairlist.py

@@ -430,7 +430,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "StaticPairList"},
       {"method": "VolatilityFilter", "lookback_days": 3,
        "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}],
-     "BTC", ['ETH/BTC', 'TKN/BTC'])
+     "BTC", ['ETH/BTC', 'TKN/BTC']),
+    # VolumePairList with no offset = unchanged pairlist
+    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
+      {"method": "OffsetFilter", "offset": 0}],
+     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']),
+    # VolumePairList with offset = 2
+    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
+      {"method": "OffsetFilter", "offset": 2}],
+     "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']),
+    # VolumePairList with higher offset, than total pairlist
+    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
+      {"method": "OffsetFilter", "offset": 100}],
+     "USDT", [])
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                       ohlcv_history, pairlists, base_currency,
@@ -728,6 +740,18 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o
     assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
 
 
+def test_OffsetFilter_error(mocker, whitelist_conf) -> None:
+    whitelist_conf['pairlists'] = (
+        [{"method": "StaticPairList"}, {"method": "OffsetFilter", "offset": -1}]
+    )
+
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    with pytest.raises(OperationalException,
+                       match=r'OffsetFilter requires offset to be >= 0'):
+        PairListManager(MagicMock, whitelist_conf)
+
+
 def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
     default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                  {'method': 'RangeStabilityFilter', 'lookback_days': 99999}]