Allow chaining of pairlists

freqtrade/freqtradebot.py

@@ -20,9 +20,9 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
 from freqtrade.persistence import Trade
-from freqtrade.resolvers import (ExchangeResolver, PairListResolver,
+from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-                                 StrategyResolver)
 from freqtrade.rpc import RPCManager, RPCMessageType
+from freqtrade.pairlist.pairlistmanager import PairListManager
 from freqtrade.state import State
 from freqtrade.strategy.interface import IStrategy, SellType
 from freqtrade.wallets import Wallets
@@ -70,8 +70,7 @@ class FreqtradeBot:
         # Attach Wallets to Strategy baseclass
         IStrategy.wallets = self.wallets
 
-        pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList')
+        self.pairlists = PairListManager(self.exchange, self.config)
-        self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist
 
         # Initializing Edge only if enabled
         self.edge = Edge(self.config, self.exchange, self.strategy) if \

freqtrade/pairlist/IPairList.py

@@ -9,25 +9,16 @@ from abc import ABC, abstractmethod
 from typing import Dict, List
 
 from freqtrade.exchange import market_is_active
-from freqtrade.pairlist.IPairListFilter import IPairListFilter
-from freqtrade.resolvers.pairlistfilter_resolver import PairListFilterResolver
 
 logger = logging.getLogger(__name__)
 
 
 class IPairList(ABC):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    def __init__(self, exchange, config, pairlistconfig: dict) -> None:
-        self._freqtrade = freqtrade
+        self._exchange = exchange
         self._config = config
-        self._whitelist = self._config['exchange']['pair_whitelist']
+        self._pairlistconfig = pairlistconfig
-        self._blacklist = self._config['exchange'].get('pair_blacklist', [])
-        self._filters = self._config.get('pairlist', {}).get('filters', {})
-        self._pairlistfilters: List[IPairListFilter] = []
-        for pl_filter in self._filters.keys():
-            self._pairlistfilters.append(
-                PairListFilterResolver(pl_filter, freqtrade, self._config).pairlistfilter
-                )
 
     @property
     def name(self) -> str:
@@ -37,22 +28,6 @@ class IPairList(ABC):
         """
         return self.__class__.__name__
 
-    @property
-    def whitelist(self) -> List[str]:
-        """
-        Has the current whitelist
-        -> no need to overwrite in subclasses
-        """
-        return self._whitelist
-
-    @property
-    def blacklist(self) -> List[str]:
-        """
-        Has the current blacklist
-        -> no need to overwrite in subclasses
-        """
-        return self._blacklist
-
     @abstractmethod
     def short_desc(self) -> str:
         """
@@ -61,28 +36,16 @@ class IPairList(ABC):
         """
 
     @abstractmethod
-    def refresh_pairlist(self) -> None:
+    def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
         """
-        Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
         -> Please overwrite in subclasses
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
         """
 
-    def validate_whitelist(self, pairlist: List[str],
-                           tickers: List[Dict] = []) -> List[str]:
-        """
-        Validate pairlist against active markets and blacklist.
-        Run PairlistFilters if these are configured.
-        """
-        pairlist = self._whitelist_for_active_markets(pairlist)
-
-        if not tickers:
-            # Refresh tickers if they are not used by the parent Pairlist
-            tickers = self._freqtrade.exchange.get_tickers()
-
-        for pl_filter in self._pairlistfilters:
-            pairlist = pl_filter.filter_pairlist(pairlist, tickers)
-        return pairlist
-
     def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary
@@ -90,16 +53,14 @@ class IPairList(ABC):
         :return: the list of pairs the user wants to trade without those unavailable or
         black_listed
         """
-        markets = self._freqtrade.exchange.markets
+        markets = self._exchange.markets
 
         sanitized_whitelist: List[str] = []
         for pair in whitelist:
-            # pair is not in the generated dynamic market, or in the blacklist ... ignore it
+            # pair is not in the generated dynamic market or has the wrong stake currency
-            if (pair in self.blacklist or pair not in markets
+            if (pair not in markets or not pair.endswith(self._config['stake_currency'])):
-                    or not pair.endswith(self._config['stake_currency'])):
                 logger.warning(f"Pair {pair} is not compatible with exchange "
-                               f"{self._freqtrade.exchange.name} or contained in "
+                               f"{self._exchange.name}. Removing it from whitelist..")
-                               f"your blacklist. Removing it from whitelist..")
                 continue
             # Check if market is active
             market = markets[pair]

freqtrade/pairlist/IPairListFilter.py

@@ -1,18 +0,0 @@
-import logging
-from abc import ABC, abstractmethod
-from typing import Dict, List
-
-logger = logging.getLogger(__name__)
-
-
-class IPairListFilter(ABC):
-
-    def __init__(self, freqtrade, config: dict) -> None:
-        self._freqtrade = freqtrade
-        self._config = config
-
-    @abstractmethod
-    def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
-        """
-        Method doing the filtering
-        """

freqtrade/pairlist/LowPriceFilter.py

@@ -2,18 +2,23 @@ import logging
 from copy import deepcopy
 from typing import Dict, List
 
-from freqtrade.pairlist.IPairListFilter import IPairListFilter
+from freqtrade.pairlist.IPairList import IPairList
 
 logger = logging.getLogger(__name__)
 
 
-class LowPriceFilter(IPairListFilter):
+class LowPriceFilter(IPairList):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    def __init__(self, exchange, config, pairlistconfig: dict) -> None:
-        super().__init__(freqtrade, config)
+        super().__init__(exchange, config, pairlistconfig)
 
-        self._low_price_percent = config['pairlist']['filters']['LowPriceFilter'].get(
+        self._low_price_percent = pairlistconfig.get('low_price_percent', 0)
-            'low_price_percent', 0)
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - Filtering pairs priced below {self._low_price_percent * 100}%."
 
     def _validate_ticker_lowprice(self, ticker) -> bool:
         """
@@ -22,7 +27,7 @@ class LowPriceFilter(IPairListFilter):
         :param precision: Precision
         :return: True if the pair can stay, false if it should be removed
         """
-        precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price']
+        precision = self._exchange.markets[ticker['symbol']]['precision']['price']
 
         compare = ticker['last'] + 1 / pow(10, precision)
         changeperc = (compare - ticker['last']) / ticker['last']
@@ -33,10 +38,14 @@ class LowPriceFilter(IPairListFilter):
         return True
 
     def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
-        """
-        Method doing the filtering
-        """
 
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
+        """
         # Copy list since we're modifying this list
         for p in deepcopy(pairlist):
             ticker = [t for t in tickers if t['symbol'] == p][0]

freqtrade/pairlist/PrecisionFilter.py

@@ -2,15 +2,18 @@ import logging
 from copy import deepcopy
 from typing import Dict, List
 
-from freqtrade.pairlist.IPairListFilter import IPairListFilter
+from freqtrade.pairlist.IPairList import IPairList
 
 logger = logging.getLogger(__name__)
 
 
-class PrecisionFilter(IPairListFilter):
+class PrecisionFilter(IPairList):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    def short_desc(self) -> str:
-        super().__init__(freqtrade, config)
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - Filtering untradable pairs."
 
     def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool:
         """
@@ -35,7 +38,7 @@ class PrecisionFilter(IPairListFilter):
 
     def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
         """
-        Method doing the filtering
+        Filters and sorts pairlists and assigns and returns them again.
         """
         if self._freqtrade.strategy.stoploss is not None:
             # Precalculate sanitized stoploss value to avoid recalculation for every pair

freqtrade/pairlist/StaticPairList.py

@@ -5,6 +5,7 @@ Provides lists as configured in config.json
 
  """
 import logging
+from typing import List, Dict
 
 from freqtrade.pairlist.IPairList import IPairList
 
@@ -13,8 +14,8 @@ logger = logging.getLogger(__name__)
 
 class StaticPairList(IPairList):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    def __init__(self, exchange, config, pairlistconfig: dict) -> None:
-        super().__init__(freqtrade, config)
+        super().__init__(exchange, config, pairlistconfig)
 
     def short_desc(self) -> str:
         """
@@ -23,8 +24,12 @@ class StaticPairList(IPairList):
         """
         return f"{self.name}: {self.whitelist}"
 
-    def refresh_pairlist(self) -> None:
+    def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
         """
-        Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
         """
-        self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist'])
+        return self.validate_whitelist(self._config['exchange']['pair_whitelist'])

freqtrade/pairlist/VolumePairList.py

@@ -5,7 +5,7 @@ Provides lists as configured in config.json
 
  """
 import logging
-from typing import List
+from typing import List, Dict
 
 from cachetools import TTLCache, cached
 
@@ -19,18 +19,17 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume']
 
 class VolumePairList(IPairList):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    def __init__(self, exchange, config, pairlistconfig: dict) -> None:
-        super().__init__(freqtrade, config)
+        super().__init__(exchange, config, pairlistconfig)
-        self._whitelistconf = self._config.get('pairlist', {}).get('config')
+
-        if 'number_assets' not in self._whitelistconf:
+        if 'number_assets' not in self._pairlistconfig:
             raise OperationalException(
                 f'`number_assets` not specified. Please check your configuration '
                 'for "pairlist.config.number_assets"')
-        self._number_pairs = self._whitelistconf['number_assets']
+        self._number_pairs = self._pairlistconfig['number_assets']
-        self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume')
+        self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
-        self._precision_filter = self._whitelistconf.get('precision_filter', True)
 
-        if not self._freqtrade.exchange.exchange_has('fetchTickers'):
+        if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
                 'Exchange does not support dynamic whitelist.'
                 'Please edit your config and restart the bot'
@@ -45,14 +44,16 @@ class VolumePairList(IPairList):
     def short_desc(self) -> str:
         """
         Short whitelist method description - used for startup-messages
-        -> Please overwrite in subclasses
         """
         return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs."
 
-    def refresh_pairlist(self) -> None:
+    def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]:
         """
-        Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively
+        Filters and sorts pairlist and returns the whitelist again.
-        -> Please overwrite in subclasses
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
         """
         # Generate dynamic whitelist
         self._whitelist = self._gen_pair_whitelist(
@@ -64,17 +65,18 @@ class VolumePairList(IPairList):
         Updates the whitelist with with a dynamically generated list
         :param base_currency: base currency as str
         :param key: sort key (defaults to 'quoteVolume')
+        :param tickers: Tickers (from exchange.get_tickers()).
         :return: List of pairs
         """
 
-        tickers = self._freqtrade.exchange.get_tickers()
+        tickers = self._exchange.get_tickers()
         # check length so that we make sure that '/' is actually in the string
         tickers = [v for k, v in tickers.items()
                    if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency
                        and v[key] is not None)]
         sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key])
         # Validate whitelist to only have active market pairs
-        pairs = self.validate_whitelist([s['symbol'] for s in sorted_tickers], tickers)
+        pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
 
         logger.info(f"Searching {self._number_pairs} pairs: {pairs[:self._number_pairs]}")
 

freqtrade/pairlist/pairlistmanager.py

@@ -0,0 +1,68 @@
+"""
+Static List provider
+
+Provides lists as configured in config.json
+
+ """
+import logging
+from copy import deepcopy
+from typing import List
+
+from freqtrade.pairlist.IPairList import IPairList
+from freqtrade.resolvers import PairListResolver
+
+logger = logging.getLogger(__name__)
+
+
+class PairListManager():
+
+    def __init__(self, exchange, config: dict) -> None:
+        self._exchange = exchange
+        self._config = config
+        self._whitelist = self._config['exchange'].get('pair_whitelist')
+        self._blacklist = self._config['exchange'].get('pair_blacklist', [])
+        self._pairlists: List[IPairList] = []
+
+        for pl in self._config.get('pairlists', [{'method': "StaticPairList"}]):
+            pairl = PairListResolver(pl.get('method'),
+                                     exchange, config,
+                                     pl.get('config')).pairlist
+            self._pairlists.append(pairl)
+
+    @property
+    def whitelist(self) -> List[str]:
+        """
+        Has the current whitelist
+        """
+        return self._whitelist
+
+    @property
+    def blacklist(self) -> List[str]:
+        """
+        Has the current blacklist
+        -> no need to overwrite in subclasses
+        """
+        return self._blacklist
+
+    def refresh_pairlist(self) -> None:
+        """
+        Run pairlist through all pairlists.
+        """
+
+        pairlist = self._whitelist.copy()
+
+        # tickers should be cached to avoid calling the exchange on each call.
+        tickers = self._exchange.get_tickers()
+        for pl in self._pairlists:
+            pl.filter_pairlist(pairlist, tickers)
+
+        pairlist = self._verify_blacklist(pairlist)
+        self._whitelist = pairlist
+
+    def _verify_blacklist(self, pairlist: List[str]) -> List[str]:
+
+        for pair in deepcopy(pairlist):
+            if pair in self.blacklist:
+                logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...")
+                pairlist.remove(pair)
+        return pairlist

freqtrade/resolvers/pairlist_resolver.py

@@ -20,13 +20,15 @@ class PairListResolver(IResolver):
 
     __slots__ = ['pairlist']
 
-    def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None:
+    def __init__(self, pairlist_name: str, exchange, config: dict, pairlistconfig) -> None:
         """
         Load the custom class from config parameter
         :param config: configuration dictionary or None
         """
-        self.pairlist = self._load_pairlist(pairlist_name, config, kwargs={'freqtrade': freqtrade,
+        self.pairlist = self._load_pairlist(pairlist_name, config,
-                                                                           'config': config})
+                                            kwargs={'exchange': exchange,
+                                                    'config': config,
+                                                    'pairlistconfig': pairlistconfig})
 
     def _load_pairlist(
             self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList:

freqtrade/resolvers/pairlistfilter_resolver.py

@@ -1,53 +0,0 @@
-# pragma pylint: disable=attribute-defined-outside-init
-
-"""
-This module load custom pairlists
-"""
-import logging
-from pathlib import Path
-
-from freqtrade import OperationalException
-from freqtrade.pairlist.IPairListFilter import IPairListFilter
-from freqtrade.resolvers import IResolver
-
-logger = logging.getLogger(__name__)
-
-
-class PairListFilterResolver(IResolver):
-    """
-    This class contains all the logic to load custom PairList class
-    """
-
-    __slots__ = ['pairlistfilter']
-
-    def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None:
-        """
-        Load the custom class from config parameter
-        :param config: configuration dictionary or None
-        """
-        self.pairlistfilter = self._load_pairlist(pairlist_name, config,
-                                                  kwargs={'freqtrade': freqtrade,
-                                                          'config': config})
-
-    def _load_pairlist(
-            self, pairlistfilter_name: str, config: dict, kwargs: dict) -> IPairListFilter:
-        """
-        Search and loads the specified pairlist.
-        :param pairlistfilter_name: name of the module to import
-        :param config: configuration dictionary
-        :param extra_dir: additional directory to search for the given pairlist
-        :return: PairList instance or None
-        """
-        current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve()
-
-        abs_paths = self.build_search_paths(config, current_path=current_path,
-                                            user_subdir=None, extra_dir=None)
-
-        pairlist = self._load_object(paths=abs_paths, object_type=IPairListFilter,
-                                     object_name=pairlistfilter_name, kwargs=kwargs)
-        if pairlist:
-            return pairlist
-        raise OperationalException(
-            f"Impossible to load PairlistFilter '{pairlistfilter_name}'. This class does not exist "
-            "or contains Python code errors."
-        )