Introduce chainable PairlistFilters

freqtrade/pairlist/IPairList.py

@@ -6,10 +6,11 @@ Provides lists as configured in config.json
  """
 import logging
 from abc import ABC, abstractmethod
-from typing import List
+from typing import Dict, List
 
 from freqtrade.exchange import market_is_active
-
+from freqtrade.pairlist.IPairListFilter import IPairListFilter
+from freqtrade.resolvers.pairlistfilter_resolver import PairListFilterResolver
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +22,12 @@ class IPairList(ABC):
         self._config = config
         self._whitelist = self._config['exchange']['pair_whitelist']
         self._blacklist = self._config['exchange'].get('pair_blacklist', [])
+        self._filters = self._config.get('pairlist', {}).get('filters', {})
+        self._pairlistfilters: List[IPairListFilter] = []
+        for pl_filter in self._filters.keys():
+            self._pairlistfilters.append(
+                PairListFilterResolver(pl_filter, freqtrade, self._config).pairlistfilter
+                )
 
     @property
     def name(self) -> str:
@@ -60,7 +67,23 @@ class IPairList(ABC):
         -> Please overwrite in subclasses
         """
 
-    def _validate_whitelist(self, whitelist: List[str]) -> List[str]:
+    def validate_whitelist(self, pairlist: List[str],
+                           tickers: List[Dict] = []) -> List[str]:
+        """
+        Validate pairlist against active markets and blacklist.
+        Run PairlistFilters if these are configured.
+        """
+        pairlist = self._whitelist_for_active_markets(pairlist)
+
+        if not tickers:
+            # Refresh tickers if they are not used by the parent Pairlist
+            tickers = self._freqtrade.exchange.get_tickers()
+
+        for pl_filter in self._pairlistfilters:
+            pairlist = pl_filter.filter_pairlist(pairlist, tickers)
+        return pairlist
+
+    def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary
         :param whitelist: the sorted list of pairs the user might want to trade
@@ -69,7 +92,7 @@ class IPairList(ABC):
         """
         markets = self._freqtrade.exchange.markets
 
-        sanitized_whitelist = set()
+        sanitized_whitelist: List[str] = []
         for pair in whitelist:
             # pair is not in the generated dynamic market, or in the blacklist ... ignore it
             if (pair in self.blacklist or pair not in markets
@@ -83,7 +106,8 @@ class IPairList(ABC):
             if not market_is_active(market):
                 logger.info(f"Ignoring {pair} from whitelist. Market is not active.")
                 continue
-            sanitized_whitelist.add(pair)
+            if pair not in sanitized_whitelist:
+                sanitized_whitelist.append(pair)
 
         # We need to remove pairs that are unknown
-        return list(sanitized_whitelist)
+        return sanitized_whitelist

freqtrade/pairlist/IPairListFilter.py

@@ -0,0 +1,18 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, List
+
+logger = logging.getLogger(__name__)
+
+
+class IPairListFilter(ABC):
+
+    def __init__(self, freqtrade, config: dict) -> None:
+        self._freqtrade = freqtrade
+        self._config = config
+
+    @abstractmethod
+    def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
+        """
+        Method doing the filtering
+        """

freqtrade/pairlist/LowPriceFilter.py

@@ -0,0 +1,48 @@
+import logging
+from copy import deepcopy
+from typing import Dict, List
+
+from freqtrade.pairlist.IPairListFilter import IPairListFilter
+
+logger = logging.getLogger(__name__)
+
+
+class LowPriceFilter(IPairListFilter):
+
+    def __init__(self, freqtrade, config: dict) -> None:
+        super().__init__(freqtrade, config)
+
+        self._low_price_percent = config['pairlist']['filters']['LowPriceFilter'].get(
+            'low_price_percent', 0)
+
+    def _validate_precision_filter_lowprice(self, ticker) -> bool:
+        """
+        Check if if one price-step is > than a certain barrier.
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param precision: Precision
+        :return: True if the pair can stay, false if it should be removed
+        """
+        precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price']
+
+        compare = ticker['last'] + 1 / pow(10, precision)
+        changeperc = (compare - ticker['last']) / ticker['last']
+        if changeperc > self._low_price_percent:
+            logger.info(f"Removed {ticker['symbol']} from whitelist, "
+                        f"because 1 unit is {changeperc * 100:.3f}%")
+            return False
+        return True
+
+    def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
+        """
+        Method doing the filtering
+        """
+
+        # Copy list since we're modifying this list
+        for p in deepcopy(pairlist):
+            ticker = [t for t in tickers if t['symbol'] == p][0]
+
+            # Filter out assets which would not allow setting a stoploss
+            if self._low_price_percent and not self._validate_precision_filter_lowprice(ticker):
+                pairlist.remove(p)
+
+        return pairlist

freqtrade/pairlist/PrecisionFilter.py

@@ -0,0 +1,51 @@
+import logging
+from copy import deepcopy
+from typing import Dict, List
+
+from freqtrade.pairlist.IPairListFilter import IPairListFilter
+
+logger = logging.getLogger(__name__)
+
+
+class PrecisionFilter(IPairListFilter):
+
+    def __init__(self, freqtrade, config: dict) -> None:
+        super().__init__(freqtrade, config)
+
+    def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool:
+        """
+        Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
+        low value pairs.
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param stoploss: stoploss value as set in the configuration
+                        (already cleaned to be 1 - stoploss)
+        :return: True if the pair can stay, false if it should be removed
+        """
+        stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss
+        # Adjust stop-prices to precision
+        sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price)
+        stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"],
+                                                                    stop_price * 0.99)
+        logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
+        if sp <= stop_gap_price:
+            logger.info(f"Removed {ticker['symbol']} from whitelist, "
+                        f"because stop price {sp} would be <= stop limit {stop_gap_price}")
+            return False
+        return True
+
+    def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
+        """
+        Method doing the filtering
+        """
+        if self._freqtrade.strategy.stoploss is not None:
+            # Precalculate sanitized stoploss value to avoid recalculation for every pair
+            stoploss = 1 - abs(self._freqtrade.strategy.stoploss)
+        # Copy list since we're modifying this list
+        for p in deepcopy(pairlist):
+            ticker = [t for t in tickers if t['symbol'] == p][0]
+            # Filter out assets which would not allow setting a stoploss
+            if (stoploss and not self._validate_precision_filter(ticker, stoploss)):
+                pairlist.remove(p)
+                continue
+
+        return pairlist

freqtrade/pairlist/StaticPairList.py

@@ -27,4 +27,4 @@ class StaticPairList(IPairList):
         """
         Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively
         """
-        self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist'])
+        self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist'])

freqtrade/pairlist/VolumePairList.py

@@ -5,7 +5,6 @@ Provides lists as configured in config.json
 
  """
 import logging
-from copy import deepcopy
 from typing import List
 
 from cachetools import TTLCache, cached
@@ -30,7 +29,6 @@ class VolumePairList(IPairList):
         self._number_pairs = self._whitelistconf['number_assets']
         self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume')
         self._precision_filter = self._whitelistconf.get('precision_filter', True)
-        self._low_price_percent_filter = self._whitelistconf.get('low_price_percent_filter', None)
 
         if not self._freqtrade.exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -60,44 +58,6 @@ class VolumePairList(IPairList):
         self._whitelist = self._gen_pair_whitelist(
             self._config['stake_currency'], self._sort_key)
 
-    def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool:
-        """
-        Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
-        low value pairs.
-        :param ticker: ticker dict as returned from ccxt.load_markets()
-        :param stoploss: stoploss value as set in the configuration
-                        (already cleaned to be 1 - stoploss)
-        :return: True if the pair can stay, false if it should be removed
-        """
-        stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss
-        # Adjust stop-prices to precision
-        sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price)
-        stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"],
-                                                                    stop_price * 0.99)
-        logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
-        if sp <= stop_gap_price:
-            logger.info(f"Removed {ticker['symbol']} from whitelist, "
-                        f"because stop price {sp} would be <= stop limit {stop_gap_price}")
-            return False
-        return True
-
-    def _validate_precision_filter_lowprice(self, ticker) -> bool:
-        """
-        Check if if one price-step is > than a certain barrier.
-        :param ticker: ticker dict as returned from ccxt.load_markets()
-        :param precision: Precision
-        :return: True if the pair can stay, false if it should be removed
-        """
-        precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price']
-
-        compare = ticker['last'] + 1 / pow(10, precision)
-        changeperc = (compare - ticker['last']) / ticker['last']
-        if changeperc > self._low_price_percent_filter:
-            logger.info(f"Removed {ticker['symbol']} from whitelist, "
-                        f"because 1 unit is {changeperc * 100:.3f}%")
-            return False
-        return True
-
     @cached(TTLCache(maxsize=1, ttl=1800))
     def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]:
         """
@@ -114,26 +74,8 @@ class VolumePairList(IPairList):
                        and v[key] is not None)]
         sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key])
         # Validate whitelist to only have active market pairs
-        valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers])
+        pairs = self.validate_whitelist([s['symbol'] for s in sorted_tickers], tickers)
-        valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs]
 
-        stoploss = None
+        logger.info(f"Searching {self._number_pairs} pairs: {pairs[:self._number_pairs]}")
-        if self._freqtrade.strategy.stoploss is not None:
-            # Precalculate sanitized stoploss value to avoid recalculation for every pair
-            stoploss = 1 - abs(self._freqtrade.strategy.stoploss)
-
-        # Copy list since we're modifying this list
-        for t in deepcopy(valid_tickers):
-            # Filter out assets which would not allow setting a stoploss
-            if (stoploss and self._precision_filter
-               and not self._validate_precision_filter(t, stoploss)):
-                valid_tickers.remove(t)
-                continue
-            if self._low_price_percent_filter and not self._validate_precision_filter_lowprice(t,):
-                valid_tickers.remove(t)
-                continue
-
-        pairs = [s['symbol'] for s in valid_tickers]
-        logger.info(f"Searching pairs: {pairs[:self._number_pairs]}")
 
         return pairs

freqtrade/resolvers/pairlistfilter_resolver.py

@@ -0,0 +1,53 @@
+# pragma pylint: disable=attribute-defined-outside-init
+
+"""
+This module load custom pairlists
+"""
+import logging
+from pathlib import Path
+
+from freqtrade import OperationalException
+from freqtrade.pairlist.IPairListFilter import IPairListFilter
+from freqtrade.resolvers import IResolver
+
+logger = logging.getLogger(__name__)
+
+
+class PairListFilterResolver(IResolver):
+    """
+    This class contains all the logic to load custom PairList class
+    """
+
+    __slots__ = ['pairlistfilter']
+
+    def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None:
+        """
+        Load the custom class from config parameter
+        :param config: configuration dictionary or None
+        """
+        self.pairlistfilter = self._load_pairlist(pairlist_name, config,
+                                                  kwargs={'freqtrade': freqtrade,
+                                                          'config': config})
+
+    def _load_pairlist(
+            self, pairlistfilter_name: str, config: dict, kwargs: dict) -> IPairListFilter:
+        """
+        Search and loads the specified pairlist.
+        :param pairlistfilter_name: name of the module to import
+        :param config: configuration dictionary
+        :param extra_dir: additional directory to search for the given pairlist
+        :return: PairList instance or None
+        """
+        current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve()
+
+        abs_paths = self.build_search_paths(config, current_path=current_path,
+                                            user_subdir=None, extra_dir=None)
+
+        pairlist = self._load_object(paths=abs_paths, object_type=IPairListFilter,
+                                     object_name=pairlistfilter_name, kwargs=kwargs)
+        if pairlist:
+            return pairlist
+        raise OperationalException(
+            f"Impossible to load PairlistFilter '{pairlistfilter_name}'. This class does not exist "
+            "or contains Python code errors."
+        )