Merge pull request #5228 from kevinjulian/agefilter-max-days-listed

feat(agefilter): add max_days_listed

docs/includes/pairlists.md

@@ -81,13 +81,13 @@ Filtering instances (not the first position in the list) will not apply any cach
 
 #### AgeFilter
 
-Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`).
+Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).
 
 When pairs are first listed on an exchange they can suffer huge price drops and volatility
 in the first few days while the pair goes through its price-discovery period. Bots can often
 be caught out buying before the pair has finished dropping in price.
 
-This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days.
+This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`.
 
 #### PerformanceFilter
 

freqtrade/plugins/pairlist/AgeFilter.py

@@ -27,6 +27,7 @@ class AgeFilter(IPairList):
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
+        self._max_days_listed = pairlistconfig.get('max_days_listed', None)
 
         if self._min_days_listed < 1:
             raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
@@ -34,6 +35,12 @@ class AgeFilter(IPairList):
             raise OperationalException("AgeFilter requires min_days_listed to not exceed "
                                        "exchange max request size "
                                        f"({exchange.ohlcv_candle_limit('1d')})")
+        if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
+            raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
+        if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
+            raise OperationalException("AgeFilter requires max_days_listed to not exceed "
+                                       "exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit('1d')})")
 
     @property
     def needstickers(self) -> bool:
@@ -48,8 +55,13 @@ class AgeFilter(IPairList):
         """
         Short whitelist method description - used for startup-messages
         """
-        return (f"{self.name} - Filtering pairs with age less than "
+        return (
-                f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
+            f"{self.name} - Filtering pairs with age less than "
+            f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
+        ) + (
+            " or more than "
+            f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
+        ) if self._max_days_listed else ''
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
@@ -61,10 +73,13 @@ class AgeFilter(IPairList):
         if not needed_pairs:
             return pairlist
 
-        since_ms = (arrow.utcnow()
+        since_days = -(
+            self._max_days_listed if self._max_days_listed else self._min_days_listed
+        ) - 1
+        since_ms = int(arrow.utcnow()
                        .floor('day')
-                         .shift(days=-self._min_days_listed - 1)
+                       .shift(days=since_days)
-                         .int_timestamp) * 1000
+                       .float_timestamp) * 1000
         candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
         if self._enabled:
             for p in deepcopy(pairlist):
@@ -86,14 +101,22 @@ class AgeFilter(IPairList):
             return True
 
         if daily_candles is not None:
-            if len(daily_candles) >= self._min_days_listed:
+            if (
+                len(daily_candles) >= self._min_days_listed
+                and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed)
+            ):
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
                 self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
                 return True
             else:
-                self.log_once(f"Removed {pair} from whitelist, because age "
+                self.log_once((
+                    f"Removed {pair} from whitelist, because age "
                     f"{len(daily_candles)} is less than {self._min_days_listed} "
-                              f"{plural(self._min_days_listed, 'day')}", logger.info)
+                    f"{plural(self._min_days_listed, 'day')}"
+                ) + ((
+                    " or more than "
+                    f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
+                ) if self._max_days_listed else ''), logger.info)
                 return False
         return False

tests/plugins/test_pairlist.py

@@ -79,7 +79,8 @@ def whitelist_conf_agefilter(default_conf):
         },
         {
             "method": "AgeFilter",
-            "min_days_listed": 2
+            "min_days_listed": 2,
+            "max_days_listed": 100
         }
     ]
     return default_conf
@@ -302,7 +303,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     # No pair for ETH, all handlers
     ([{"method": "StaticPairList"},
       {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "AgeFilter", "min_days_listed": 2},
+      {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": None},
       {"method": "PrecisionFilter"},
       {"method": "PriceFilter", "low_price_ratio": 0.03},
       {"method": "SpreadFilter", "max_spread_ratio": 0.005},
@@ -310,12 +311,24 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
      "ETH", []),
     # AgeFilter and VolumePairList (require 2 days only, all should pass age test)
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "AgeFilter", "min_days_listed": 2}],
+      {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": 100}],
      "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
     # AgeFilter and VolumePairList (require 10 days, all should fail age test)
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "AgeFilter", "min_days_listed": 10}],
+      {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": None}],
      "BTC", []),
+    # AgeFilter and VolumePairList (all pair listed > 2, all should fail age test)
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}],
+     "BTC", []),
+    # AgeFilter and VolumePairList LTC/BTC has 6 candles - removes all
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 5}],
+     "BTC", []),
+    # AgeFilter and VolumePairList LTC/BTC has 6 candles - passes
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 10}],
+     "BTC", ["LTC/BTC"]),
     # Precisionfilter and quote volume
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PrecisionFilter"}],
@@ -431,7 +444,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
     ohlcv_data = {
         ('ETH/BTC', '1d'): ohlcv_history,
         ('TKN/BTC', '1d'): ohlcv_history,
-        ('LTC/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history.append(ohlcv_history),
         ('XRP/BTC', '1d'): ohlcv_history,
         ('HOT/BTC', '1d'): ohlcv_history_high_vola,
     }
@@ -480,9 +493,13 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
 
         for pairlist in pairlists:
             if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
-                    len(ohlcv_history) <= pairlist['min_days_listed']:
+                    len(ohlcv_history) < pairlist['min_days_listed']:
                 assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
                                   r'.* day.*', caplog)
+            if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \
+                    len(ohlcv_history) > pairlist['max_days_listed']:
+                assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
+                                  r'.* day.* or more than .* day', caplog)
             if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
                 assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
                                   r'would be <= stop limit.*', caplog)
@@ -650,6 +667,22 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick
         get_patched_freqtradebot(mocker, default_conf)
 
 
+def test_agefilter_max_days_lower_than_min_days(mocker, default_conf, markets, tickers):
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'AgeFilter', 'min_days_listed': 3,
+                                 "max_days_listed": 2}]
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+
+    with pytest.raises(OperationalException,
+                       match=r'AgeFilter max_days_listed <= min_days_listed not permitted'):
+        get_patched_freqtradebot(mocker, default_conf)
+
+
 def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers):
     default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                  {'method': 'AgeFilter', 'min_days_listed': 99999}]