Merge pull request #3521 from gambcl/develop

AgeFilter for filtering out newly listed pairs

config_full.json.example

@@ -64,6 +64,7 @@
             "sort_key": "quoteVolume",
             "refresh_period": 1800
         },
+        {"method": "AgeFilter", "min_days_listed": 10},
         {"method": "PrecisionFilter"},
         {"method": "PriceFilter", "low_price_ratio": 0.01},
         {"method": "SpreadFilter", "max_spread_ratio": 0.005}

docs/configuration.md

@@ -592,7 +592,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade.
 
 In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler).
 
-Additionaly, [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
+Additionaly, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
 
 If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
 
@@ -602,6 +602,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac
 
 * [`StaticPairList`](#static-pair-list) (default, if not configured differently)
 * [`VolumePairList`](#volume-pair-list)
+* [`AgeFilter`](#agefilter)
 * [`PrecisionFilter`](#precisionfilter)
 * [`PriceFilter`](#pricefilter)
 * [`ShuffleFilter`](#shufflefilter)
@@ -645,6 +646,16 @@ The `refresh_period` setting allows to define the period (in seconds), at which
 }],
 ```
 
+#### AgeFilter
+
+Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`).
+
+When pairs are first listed on an exchange they can suffer huge price drops and volatility
+in the first few days while the pair goes through its price-discovery period. Bots can often
+be caught out buying before the pair has finished dropping in price.
+
+This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days.
+
 #### PrecisionFilter
 
 Filters low-value coins which would not allow setting stoplosses.
@@ -692,6 +703,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
         "number_assets": 20,
         "sort_key": "quoteVolume",
     },
+    {"method": "AgeFilter", "min_days_listed": 10},
     {"method": "PrecisionFilter"},
     {"method": "PriceFilter", "low_price_ratio": 0.01},
     {"method": "SpreadFilter", "max_spread_ratio": 0.005},

freqtrade/constants.py

@@ -22,7 +22,8 @@ ORDERBOOK_SIDES = ['ask', 'bid']
 ORDERTYPE_POSSIBILITIES = ['limit', 'market']
 ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
-                       'PrecisionFilter', 'PriceFilter', 'ShuffleFilter', 'SpreadFilter']
+                       'AgeFilter', 'PrecisionFilter', 'PriceFilter',
+                       'ShuffleFilter', 'SpreadFilter']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
 DRY_RUN_WALLET = 1000
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons

freqtrade/pairlist/AgeFilter.py

@@ -0,0 +1,76 @@
+"""
+Minimum age (days listed) pair list filter
+"""
+import logging
+import arrow
+from typing import Any, Dict
+
+from freqtrade.misc import plural
+from freqtrade.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class AgeFilter(IPairList):
+
+    # Checked symbols cache (dictionary of ticker symbol => timestamp)
+    _symbolsChecked: Dict[str, int] = {}
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
+        self._enabled = self._min_days_listed >= 1
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return True
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return (f"{self.name} - Filtering pairs with age less than "
+                f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
+
+    def _validate_pair(self, ticker: dict) -> bool:
+        """
+        Validate age for the ticker
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :return: True if the pair can stay, False if it should be removed
+        """
+
+        # Check symbol in cache
+        if ticker['symbol'] in self._symbolsChecked:
+            return True
+
+        since_ms = int(arrow.utcnow()
+                       .floor('day')
+                       .shift(days=-self._min_days_listed)
+                       .float_timestamp) * 1000
+
+        daily_candles = self._exchange.get_historic_ohlcv(pair=ticker['symbol'],
+                                                          timeframe='1d',
+                                                          since_ms=since_ms)
+
+        if daily_candles is not None:
+            if len(daily_candles) > self._min_days_listed:
+                # We have fetched at least the minimum required number of daily candles
+                # Add to cache, store the time we last checked this symbol
+                self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000
+                return True
+            else:
+                self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
+                                                 f"because age is less than "
+                                                 f"{self._min_days_listed} "
+                                                 f"{plural(self._min_days_listed, 'day')}")
+                return False
+        return False

freqtrade/pairlist/IPairList.py

@@ -68,7 +68,7 @@ class IPairList(ABC):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
 

freqtrade/pairlist/PrecisionFilter.py

@@ -27,7 +27,7 @@ class PrecisionFilter(IPairList):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
         return True

freqtrade/pairlist/PriceFilter.py

@@ -24,7 +24,7 @@ class PriceFilter(IPairList):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
         return True

freqtrade/pairlist/ShuffleFilter.py

@@ -25,7 +25,7 @@ class ShuffleFilter(IPairList):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
         return False

freqtrade/pairlist/SpreadFilter.py

@@ -24,7 +24,7 @@ class SpreadFilter(IPairList):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
         return True

freqtrade/pairlist/StaticPairList.py

@@ -28,7 +28,7 @@ class StaticPairList(IPairList):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
         return False

freqtrade/pairlist/VolumePairList.py

@@ -54,7 +54,7 @@ class VolumePairList(IPairList):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
         return True

tests/pairlist/test_pairlist.py

@@ -57,6 +57,31 @@ def whitelist_conf_2(default_conf):
     return default_conf
 
 
+@pytest.fixture(scope="function")
+def whitelist_conf_3(default_conf):
+    default_conf['stake_currency'] = 'BTC'
+    default_conf['exchange']['pair_whitelist'] = [
+        'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC',
+        'BTT/BTC', 'HOT/BTC', 'FUEL/BTC', 'XRP/BTC'
+    ]
+    default_conf['exchange']['pair_blacklist'] = [
+        'BLK/BTC'
+    ]
+    default_conf['pairlists'] = [
+        {
+            "method": "VolumePairList",
+            "number_assets": 5,
+            "sort_key": "quoteVolume",
+            "refresh_period": 0,
+        },
+        {
+            "method": "AgeFilter",
+            "min_days_listed": 2
+        }
+    ]
+    return default_conf
+
+
 @pytest.fixture(scope="function")
 def static_pl_conf(whitelist_conf):
     whitelist_conf['pairlists'] = [
@@ -220,11 +245,20 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     # No pair for ETH, all handlers
     ([{"method": "StaticPairList"},
       {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 2},
       {"method": "PrecisionFilter"},
       {"method": "PriceFilter", "low_price_ratio": 0.03},
       {"method": "SpreadFilter", "max_spread_ratio": 0.005},
       {"method": "ShuffleFilter"}],
      "ETH", []),
+    # AgeFilter and VolumePairList (require 2 days only, all should pass age test)
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 2}],
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
+    # AgeFilter and VolumePairList (require 10 days, all should fail age test)
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 10}],
+     "BTC", []),
     # Precisionfilter and quote volume
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PrecisionFilter"}],
@@ -272,7 +306,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     # ShuffleFilter, no seed
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "ShuffleFilter"}],
-     "USDT", 3),  # whitelist_result is integer -- check only lenght of randomized pairlist
+     "USDT", 3),  # whitelist_result is integer -- check only length of randomized pairlist
+    # AgeFilter only
+    ([{"method": "AgeFilter", "min_days_listed": 2}],
+     "BTC", 'filter_at_the_beginning'),  # OperationalException expected
     # PrecisionFilter after StaticPairList
     ([{"method": "StaticPairList"},
       {"method": "PrecisionFilter"}],
@@ -307,8 +344,8 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
      "BTC", 'static_in_the_middle'),
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
-                                      pairlists, base_currency, whitelist_result,
-                                      caplog) -> None:
+                                      ohlcv_history_list, pairlists, base_currency,
+                                      whitelist_result, caplog) -> None:
     whitelist_conf['pairlists'] = pairlists
     whitelist_conf['stake_currency'] = base_currency
 
@@ -324,7 +361,11 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
     mocker.patch.multiple('freqtrade.exchange.Exchange',
                           get_tickers=tickers,
-                          markets=PropertyMock(return_value=shitcoinmarkets),
+                          markets=PropertyMock(return_value=shitcoinmarkets)
+                          )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
     )
 
     # Set whitelist_result to None if pairlist is invalid and should produce exception
@@ -346,6 +387,10 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
             len(whitelist) == whitelist_result
 
         for pairlist in pairlists:
+            if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
+                    len(ohlcv_history_list) <= pairlist['min_days_listed']:
+                assert log_has_re(r'^Removed .* from whitelist, because age is less than '
+                                  r'.* day.*', caplog)
             if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
                 assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
                                   r'would be <= stop limit.*', caplog)
@@ -468,6 +513,29 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
     assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
 
 
+def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_history_list):
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
+    )
+
+    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_3)
+    assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
+    freqtrade.pairlists.refresh_pairlist()
+    assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
+
+    previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count
+    freqtrade.pairlists.refresh_pairlist()
+    # Should not have increased since first call.
+    assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
+
+
 def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog):
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))