Merge pull request #2442 from freqtrade/volumeList_enhanced_filter

Pairlists enhanced filter options

config.json.example

@@ -52,6 +52,9 @@
             "DOGE/BTC"
         ]
     },
+    "pairlists": [
+        {"method": "StaticPairList"}
+    ],
     "edge": {
         "enabled": false,
         "process_throttle_secs": 3600,
@@ -68,7 +71,7 @@
         "remove_pumps": false
     },
     "telegram": {
-        "enabled": true,
+        "enabled": false,
         "token": "your_telegram_token",
         "chat_id": "your_telegram_chat_id"
     },

config_binance.json.example

@@ -54,6 +54,9 @@
             "BNB/BTC"
         ]
     },
+    "pairlists": [
+        {"method": "StaticPairList"}
+    ],
     "edge": {
         "enabled": false,
         "process_throttle_secs": 3600,

config_full.json.example

@@ -50,14 +50,18 @@
         "buy": "gtc",
         "sell": "gtc"
     },
-    "pairlist": {
+    "pairlists": [
+        {"method": "StaticPairList"},
+        {
             "method": "VolumePairList",
-        "config": {
             "number_assets": 20,
             "sort_key": "quoteVolume",
-            "precision_filter": false
+            "refresh_period": 1800
-        }
         },
+        {"method": "PrecisionFilter"},
+        {"method": "PriceFilter", "low_price_ratio": 0.01
+        }
+    ],
     "exchange": {
         "name": "bittrex",
         "sandbox": false,

config_kraken.json.example

@@ -46,6 +46,9 @@
 
         ]
     },
+    "pairlists": [
+        {"method": "StaticPairList"}
+    ],
     "edge": {
         "enabled": false,
         "process_throttle_secs": 3600,

docs/configuration.md

@@ -82,8 +82,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded.
 | `edge` | false | Please refer to [edge configuration document](edge.md) for detailed explanation.
 | `experimental.block_bad_exchanges` | true | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
-| `pairlist.method` | StaticPairList | Use static or dynamic volume-based pairlist. [More information below](#dynamic-pairlists).
+| `pairlists` | StaticPairList | Define one or more pairlists to be used. [More information below](#dynamic-pairlists).
-| `pairlist.config` | None | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists).
 | `telegram.enabled` | true | **Required.** Enable or not the usage of Telegram.
 | `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.***
 | `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.***
@@ -362,7 +361,7 @@ For example, to test the order type `FOK` with Kraken, and modify candle_limit t
 The `fiat_display_currency` configuration parameter sets the base currency to use for the
 conversion from coin to fiat in the bot Telegram reports.
 
-The valid values are:p
+The valid values are:
 
 ```json
 "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD"
@@ -376,6 +375,88 @@ The valid values are:
 "BTC", "ETH", "XRP", "LTC", "BCH", "USDT"
 ```
 
+## Pairlists
+
+Pairlists define the list of pairs that the bot should trade.
+There are [`StaticPairList`](#static-pair-list) and dynamic Whitelists available.
+
+[`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter) act as filters, removing low-value pairs.
+
+All pairlists can be chained, and a combination of all pairlists will become your new whitelist. Pairlists are executed in the sequence they are configured. You should always configure either `StaticPairList` or `DynamicPairList` as starting pairlists.
+
+Inactive markets and blacklisted pairs are always removed from the resulting `pair_whitelist`.
+
+### Available Pairlists
+
+* [`StaticPairList`](#static-pair-list) (default, if not configured differently)
+* [`VolumePairList`](#volume-pair-list)
+* [`PrecisionFilter`](#precision-filter)
+* [`PriceFilter`](#price-pair-filter)
+
+#### Static Pair List
+
+By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. 
+
+It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`.
+
+```json
+"pairlists": [
+    {"method": "StaticPairList"}
+    ],
+```
+
+#### Volume Pair List
+
+`VolumePairList` selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume` and defaults to `quoteVolume`.
+
+`VolumePairList` considers outputs of previous pairlists unless  it's the first configured pairlist, it does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange.
+
+`refresh_period` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
+
+```json
+"pairlists": [{
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "refresh_period": 1800,
+],
+```
+
+#### Precision Filter
+
+Filters low-value coins which would not allow setting a stoploss.
+
+#### Price Pair Filter
+
+The `PriceFilter` allows filtering of pairs by price.
+Currently, only `low_price_ratio` is implemented, where a raise of 1 price unit (pip) is below the `low_price_ratio` ratio.
+This option is disabled by default, and will only apply if set to <> 0.
+
+Calculation example:  
+Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value. 
+
+These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses.
+
+### Full Pairlist example
+
+The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter), filtering all assets where 1 priceunit is > 1%.
+
+```json
+"exchange": {
+    "pair_whitelist": [],
+    "pair_blacklist": ["BNB/BTC"]
+},
+"pairlists": [
+    {
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+    },
+    {"method": "PrecisionFilter"},
+    {"method": "PriceFilter", "low_price_ratio": 0.01}
+    ],
+```
+
 ## Switch to Dry-run mode
 
 We recommend starting the bot in the Dry-run mode to see how your bot will
@@ -405,45 +486,6 @@ creating trades on the exchange.
 Once you will be happy with your bot performance running in the Dry-run mode,
 you can switch it to production mode.
 
-### Dynamic Pairlists
-
-Dynamic pairlists select pairs for you based on the logic configured.
-The bot runs against all pairs (with that stake) on the exchange, and a number of assets
-(`number_assets`) is selected based on the selected criteria.
-
-By default, the `StaticPairList` method is used.
-The Pairlist method is configured as `pair_whitelist` parameter under the `exchange`
-section of the configuration.
-
-**Available Pairlist methods:**
-
-* `StaticPairList`
-  * It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`.
-* `VolumePairList`
-  * It selects `number_assets` top pairs based on `sort_key`, which can be one of
-`askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`.
-  * There is a possibility to filter low-value coins that would not allow setting a stop loss
-(set `precision_filter` parameter to `true` for this).
-  * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration.
-  * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match.
-
-Example:
-
-```json
-"exchange": {
-    "pair_whitelist": [],
-    "pair_blacklist": ["BNB/BTC"]
-},
-"pairlist": {
-        "method": "VolumePairList",
-        "config": {
-            "number_assets": 20,
-            "sort_key": "quoteVolume",
-            "precision_filter": false
-        }
-    },
-```
-
 ## Switch to production mode
 
 In production mode, the bot will engage your money. Be careful, since a wrong
@@ -476,7 +518,7 @@ you run it in production mode.
 
 You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange.
 
-### Using proxy with FreqTrade
+### Using proxy with Freqtrade
 
 To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration.
 
@@ -496,14 +538,13 @@ export HTTPS_PROXY="http://addr:port"
 freqtrade
 ```
 
-
+## Embedding Strategies
-### Embedding Strategies
 
 FreqTrade provides you with with an easy way to embed the strategy into your configuration file.
 This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field,
 in your chosen config file.
 
-#### Encoding a string as BASE64
+### Encoding a string as BASE64
 
 This is a quick example, how to generate the BASE64 string in python
 

docs/developer.md

@@ -46,15 +46,18 @@ def test_method_to_test(caplog):
 The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine.
 
 #### Install
+
 * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
 * [docker](https://docs.docker.com/install/)
 * [docker-compose](https://docs.docker.com/compose/install/)
 
 #### Starting the bot
 ##### Use the develop dockerfile
+
 ``` bash
 rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml
 ```
+
 #### Docker Compose
 
 ##### Starting
@@ -62,9 +65,11 @@ rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml
 ``` bash
 docker-compose up
 ```
+
 ![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png)
 
 ##### Rebuilding
+
 ``` bash
 docker-compose build
 ```
@@ -77,8 +82,8 @@ that can be effected by `docker-compose up` or `docker-compose run freqtrade_dev
 ``` bash
 docker-compose exec freqtrade_develop /bin/bash
 ```
-![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)
 
+![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)
 
 ## Modules
 
@@ -95,22 +100,22 @@ This is a simple provider, which however serves as a good example on how to star
 
 Next, modify the classname of the provider (ideally align this with the Filename).
 
-The base-class provides the an instance of the bot (`self._freqtrade`), as well as the configuration (`self._config`), and initiates both `_blacklist` and `_whitelist`.
+The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists.
 
 ```python
-        self._freqtrade = freqtrade
+        self._exchange = exchange
+        self._pairlistmanager = pairlistmanager
         self._config = config
-        self._whitelist = self._config['exchange']['pair_whitelist']
+        self._pairlistconfig = pairlistconfig
-        self._blacklist = self._config['exchange'].get('pair_blacklist', [])
+        self._pairlist_pos = pairlist_pos
 ```
 
-
 Now, let's step through the methods which require actions:
 
-#### configuration
+#### Pairlist configuration
 
 Configuration for PairListProvider is done in the bot configuration file in the element `"pairlist"`.
-This Pairlist-object may contain a `"config"` dict with additional configurations for the configured pairlist.
+This Pairlist-object may contain configurations with additional configurations for the configured pairlist.
 By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the whitelist. Please follow this to ensure a consistent user experience.
 
 Additional elements can be configured as needed. `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic.
@@ -120,29 +125,30 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke
 Returns a description used for Telegram messages.
 This should contain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`.
 
-#### refresh_pairlist
+#### filter_pairlist
 
 Override this method and run all calculations needed in this method.
 This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations.
 
-Assign the resulting whiteslist to `self._whitelist` and `self._blacklist` respectively. These will then be used to run the bot in this iteration. Pairs with open trades will be added to the whitelist to have the sell-methods run correctly.
+It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`.
 
-Please also run `self._validate_whitelist(pairs)` and to check and remove pairs with inactive markets. This function is available in the Parent class (`StaticPairList`) and should ideally not be overwritten.
+It must return the resulting pairlist (which may then be passed into the next pairlist filter).
+
+Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected.
 
 ##### sample
 
 ``` python
-    def refresh_pairlist(self) -> None:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         # Generate dynamic whitelist
-        pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key)
+        pairs = self._calculate_pairlist(pairlist, tickers)
-        # Validate whitelist to only have active market pairs
+        return pairs
-        self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs]
 ```
 
 #### _gen_pair_whitelist
 
 This is a simple method used by `VolumePairList` - however serves as a good example.
-It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider.
+In VolumePairList, this implements different methods of sorting, does early validation so only the expected number of pairs is returned.
 
 ## Implement a new Exchange (WIP)
 

freqtrade/configuration/config_validation.py

@@ -122,6 +122,7 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None:
                                               RunMode.UTIL_NO_EXCHANGE, RunMode.UTIL_EXCHANGE]:
         return
 
-    if (conf.get('pairlist', {}).get('method', 'StaticPairList') == 'StaticPairList'
+    for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]):
+        if (pl.get('method') == 'StaticPairList'
                 and not conf.get('exchange', {}).get('pair_whitelist')):
             raise OperationalException("StaticPairList requires pair_whitelist to be set.")

freqtrade/configuration/configuration.py

@@ -81,6 +81,9 @@ class Configuration:
         if 'ask_strategy' not in config:
             config['ask_strategy'] = {}
 
+        if 'pairlists' not in config:
+            config['pairlists'] = []
+
         # validate configuration before returning
         logger.info('Validating configuration ...')
         validate_config_schema(config)

freqtrade/configuration/deprecated_settings.py

@@ -57,3 +57,19 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
                                'experimental', 'sell_profit_only')
     process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
                                'experimental', 'ignore_roi_if_buy_signal')
+
+    if config.get('pairlist', {}).get("method") == 'VolumePairList':
+        logger.warning(
+            "DEPRECATED: "
+            f"Using VolumePairList in pairlist is deprecated and must be moved to pairlists. "
+            "Please refer to the docs on configuration details")
+        pl = {'method': 'VolumePairList'}
+        pl.update(config.get('pairlist', {}).get('config'))
+        config['pairlists'].append(pl)
+
+    if config.get('pairlist', {}).get('config', {}).get('precision_filter'):
+        logger.warning(
+            "DEPRECATED: "
+            f"Using precision_filter setting is deprecated and has been replaced by"
+            "PrecisionFilter. Please refer to the docs on configuration details")
+        config['pairlists'].append({'method': 'PrecisionFilter'})

freqtrade/constants.py

@@ -18,7 +18,7 @@ REQUIRED_ORDERTIF = ['buy', 'sell']
 REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']
 ORDERTYPE_POSSIBILITIES = ['limit', 'market']
 ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
-AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList']
+AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter']
 DRY_RUN_WALLET = 999.9
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
 
@@ -149,13 +149,16 @@ CONF_SCHEMA = {
                 'block_bad_exchanges': {'type': 'boolean'}
             }
         },
-        'pairlist': {
+        'pairlists': {
+            'type': 'array',
+            'items': {
                 'type': 'object',
                 'properties': {
                     'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS},
                     'config': {'type': 'object'}
                 },
-            'required': ['method']
+                'required': ['method'],
+            }
         },
         'telegram': {
             'type': 'object',

freqtrade/freqtradebot.py

@@ -20,9 +20,9 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
 from freqtrade.persistence import Trade
-from freqtrade.resolvers import (ExchangeResolver, PairListResolver,
+from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-                                 StrategyResolver)
 from freqtrade.rpc import RPCManager, RPCMessageType
+from freqtrade.pairlist.pairlistmanager import PairListManager
 from freqtrade.state import State
 from freqtrade.strategy.interface import IStrategy, SellType
 from freqtrade.wallets import Wallets
@@ -70,8 +70,7 @@ class FreqtradeBot:
         # Attach Wallets to Strategy baseclass
         IStrategy.wallets = self.wallets
 
-        pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList')
+        self.pairlists = PairListManager(self.exchange, self.config)
-        self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist
 
         # Initializing Edge only if enabled
         self.edge = Edge(self.config, self.exchange, self.strategy) if \

freqtrade/pairlist/IPairList.py

@@ -5,22 +5,31 @@ Provides lists as configured in config.json
 
  """
 import logging
-from abc import ABC, abstractmethod
+from abc import ABC, abstractmethod, abstractproperty
-from typing import List
+from copy import deepcopy
+from typing import Dict, List
 
 from freqtrade.exchange import market_is_active
 
-
 logger = logging.getLogger(__name__)
 
 
 class IPairList(ABC):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict,
-        self._freqtrade = freqtrade
+                 pairlist_pos: int) -> None:
+        """
+        :param exchange: Exchange instance
+        :param pairlistmanager: Instanciating Pairlist manager
+        :param config: Global bot configuration
+        :param pairlistconfig: Configuration for this pairlist - can be empty.
+        :param pairlist_pos: Position of the filter in the pairlist-filter-list
+        """
+        self._exchange = exchange
+        self._pairlistmanager = pairlistmanager
         self._config = config
-        self._whitelist = self._config['exchange']['pair_whitelist']
+        self._pairlistconfig = pairlistconfig
-        self._blacklist = self._config['exchange'].get('pair_blacklist', [])
+        self._pairlist_pos = pairlist_pos
 
     @property
     def name(self) -> str:
@@ -30,21 +39,13 @@ class IPairList(ABC):
         """
         return self.__class__.__name__
 
-    @property
+    @abstractproperty
-    def whitelist(self) -> List[str]:
+    def needstickers(self) -> bool:
         """
-        Has the current whitelist
+        Boolean property defining if tickers are necessary.
-        -> no need to overwrite in subclasses
+        If no Pairlist requries tickers, an empty List is passed
+        as tickers argument to filter_pairlist
         """
-        return self._whitelist
-
-    @property
-    def blacklist(self) -> List[str]:
-        """
-        Has the current blacklist
-        -> no need to overwrite in subclasses
-        """
-        return self._blacklist
 
     @abstractmethod
     def short_desc(self) -> str:
@@ -54,36 +55,62 @@ class IPairList(ABC):
         """
 
     @abstractmethod
-    def refresh_pairlist(self) -> None:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
-        Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
         -> Please overwrite in subclasses
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
         """
 
-    def _validate_whitelist(self, whitelist: List[str]) -> List[str]:
+    @staticmethod
+    def verify_blacklist(pairlist: List[str], blacklist: List[str]) -> List[str]:
+        """
+        Verify and remove items from pairlist - returning a filtered pairlist.
+        """
+        for pair in deepcopy(pairlist):
+            if pair in blacklist:
+                logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...")
+                pairlist.remove(pair)
+        return pairlist
+
+    def _verify_blacklist(self, pairlist: List[str]) -> List[str]:
+        """
+        Proxy method to verify_blacklist for easy access for child classes.
+        """
+        return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist)
+
+    def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary
         :param whitelist: the sorted list of pairs the user might want to trade
         :return: the list of pairs the user wants to trade without those unavailable or
         black_listed
         """
-        markets = self._freqtrade.exchange.markets
+        markets = self._exchange.markets
 
-        sanitized_whitelist = set()
+        sanitized_whitelist: List[str] = []
-        for pair in whitelist:
+        for pair in pairlist:
-            # pair is not in the generated dynamic market, or in the blacklist ... ignore it
+            # pair is not in the generated dynamic market or has the wrong stake currency
-            if (pair in self.blacklist or pair not in markets
+            if pair not in markets:
-                    or not pair.endswith(self._config['stake_currency'])):
                 logger.warning(f"Pair {pair} is not compatible with exchange "
-                               f"{self._freqtrade.exchange.name} or contained in "
+                               f"{self._exchange.name}. Removing it from whitelist..")
-                               f"your blacklist. Removing it from whitelist..")
                 continue
+            if not pair.endswith(self._config['stake_currency']):
+                logger.warning(f"Pair {pair} is not compatible with your stake currency "
+                               f"{self._config['stake_currency']}. Removing it from whitelist..")
+                continue
+
             # Check if market is active
             market = markets[pair]
             if not market_is_active(market):
                 logger.info(f"Ignoring {pair} from whitelist. Market is not active.")
                 continue
-            sanitized_whitelist.add(pair)
+            if pair not in sanitized_whitelist:
+                sanitized_whitelist.append(pair)
 
+        sanitized_whitelist = self._verify_blacklist(sanitized_whitelist)
         # We need to remove pairs that are unknown
-        return list(sanitized_whitelist)
+        return sanitized_whitelist

freqtrade/pairlist/PrecisionFilter.py

@@ -0,0 +1,62 @@
+import logging
+from copy import deepcopy
+from typing import Dict, List
+
+from freqtrade.pairlist.IPairList import IPairList
+
+logger = logging.getLogger(__name__)
+
+
+class PrecisionFilter(IPairList):
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requries tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return True
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - Filtering untradable pairs."
+
+    def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool:
+        """
+        Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
+        low value pairs.
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param stoploss: stoploss value as set in the configuration
+                        (already cleaned to be 1 - stoploss)
+        :return: True if the pair can stay, false if it should be removed
+        """
+        stop_price = ticker['ask'] * stoploss
+        # Adjust stop-prices to precision
+        sp = self._exchange.symbol_price_prec(ticker["symbol"], stop_price)
+        stop_gap_price = self._exchange.symbol_price_prec(ticker["symbol"], stop_price * 0.99)
+        logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
+        if sp <= stop_gap_price:
+            logger.info(f"Removed {ticker['symbol']} from whitelist, "
+                        f"because stop price {sp} would be <= stop limit {stop_gap_price}")
+            return False
+        return True
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Filters and sorts pairlists and assigns and returns them again.
+        """
+        if self._config.get('stoploss') is not None:
+            # Precalculate sanitized stoploss value to avoid recalculation for every pair
+            stoploss = 1 - abs(self._config.get('stoploss'))
+        # Copy list since we're modifying this list
+        for p in deepcopy(pairlist):
+            ticker = tickers.get(p)
+            # Filter out assets which would not allow setting a stoploss
+            if not ticker or (stoploss and not self._validate_precision_filter(ticker, stoploss)):
+                pairlist.remove(p)
+                continue
+
+        return pairlist

freqtrade/pairlist/PriceFilter.py

@@ -0,0 +1,69 @@
+import logging
+from copy import deepcopy
+from typing import Dict, List
+
+from freqtrade.pairlist.IPairList import IPairList
+
+logger = logging.getLogger(__name__)
+
+
+class PriceFilter(IPairList):
+
+    def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict,
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0)
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requries tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return True
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - Filtering pairs priced below {self._low_price_ratio * 100}%."
+
+    def _validate_ticker_lowprice(self, ticker) -> bool:
+        """
+        Check if if one price-step (pip) is > than a certain barrier.
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param precision: Precision
+        :return: True if the pair can stay, false if it should be removed
+        """
+        precision = self._exchange.markets[ticker['symbol']]['precision']['price']
+
+        compare = ticker['last'] + 1 / pow(10, precision)
+        changeperc = (compare - ticker['last']) / ticker['last']
+        if changeperc > self._low_price_ratio:
+            logger.info(f"Removed {ticker['symbol']} from whitelist, "
+                        f"because 1 unit is {changeperc * 100:.3f}%")
+            return False
+        return True
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
+        """
+        # Copy list since we're modifying this list
+        for p in deepcopy(pairlist):
+            ticker = tickers.get(p)
+            if not ticker:
+                pairlist.remove(p)
+
+            # Filter out assets which would not allow setting a stoploss
+            if self._low_price_ratio and not self._validate_ticker_lowprice(ticker):
+                pairlist.remove(p)
+
+        return pairlist

freqtrade/pairlist/StaticPairList.py

@@ -5,6 +5,7 @@ Provides lists as configured in config.json
 
  """
 import logging
+from typing import Dict, List
 
 from freqtrade.pairlist.IPairList import IPairList
 
@@ -13,18 +14,28 @@ logger = logging.getLogger(__name__)
 
 class StaticPairList(IPairList):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    @property
-        super().__init__(freqtrade, config)
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requries tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
 
     def short_desc(self) -> str:
         """
         Short whitelist method description - used for startup-messages
         -> Please overwrite in subclasses
         """
-        return f"{self.name}: {self.whitelist}"
+        return f"{self.name}"
 
-    def refresh_pairlist(self) -> None:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
-        Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
         """
-        self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist'])
+        return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])

freqtrade/pairlist/VolumePairList.py

@@ -5,11 +5,12 @@ Provides lists as configured in config.json
 
  """
 import logging
-from typing import List
+from datetime import datetime
-from cachetools import TTLCache, cached
+from typing import Dict, List
 
-from freqtrade.pairlist.IPairList import IPairList
 from freqtrade import OperationalException
+from freqtrade.pairlist.IPairList import IPairList
+
 logger = logging.getLogger(__name__)
 
 SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume']
@@ -17,18 +18,19 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume']
 
 class VolumePairList(IPairList):
 
-    def __init__(self, freqtrade, config: dict) -> None:
+    def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict,
-        super().__init__(freqtrade, config)
+                 pairlist_pos: int) -> None:
-        self._whitelistconf = self._config.get('pairlist', {}).get('config')
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
-        if 'number_assets' not in self._whitelistconf:
+
+        if 'number_assets' not in self._pairlistconfig:
             raise OperationalException(
                 f'`number_assets` not specified. Please check your configuration '
                 'for "pairlist.config.number_assets"')
-        self._number_pairs = self._whitelistconf['number_assets']
+        self._number_pairs = self._pairlistconfig['number_assets']
-        self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume')
+        self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
-        self._precision_filter = self._whitelistconf.get('precision_filter', False)
+        self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
 
-        if not self._freqtrade.exchange.exchange_has('fetchTickers'):
+        if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
                 'Exchange does not support dynamic whitelist.'
                 'Please edit your config and restart the bot'
@@ -36,6 +38,16 @@ class VolumePairList(IPairList):
         if not self._validate_keys(self._sort_key):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
+        self._last_refresh = 0
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requries tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return True
 
     def _validate_keys(self, key):
         return key in SORT_VALUES
@@ -43,54 +55,54 @@ class VolumePairList(IPairList):
     def short_desc(self) -> str:
         """
         Short whitelist method description - used for startup-messages
-        -> Please overwrite in subclasses
         """
-        return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs."
+        return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
 
-    def refresh_pairlist(self) -> None:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
-        Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively
+        Filters and sorts pairlist and returns the whitelist again.
-        -> Please overwrite in subclasses
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
         """
         # Generate dynamic whitelist
-        self._whitelist = self._gen_pair_whitelist(
+        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
-            self._config['stake_currency'], self._sort_key)
+            self._last_refresh = int(datetime.now().timestamp())
+            return self._gen_pair_whitelist(pairlist,
+                                            tickers,
+                                            self._config['stake_currency'],
+                                            self._sort_key,
+                                            )
+        else:
+            return pairlist
 
-    @cached(TTLCache(maxsize=1, ttl=1800))
+    def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, key: str) -> List[str]:
-    def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]:
         """
         Updates the whitelist with with a dynamically generated list
         :param base_currency: base currency as str
         :param key: sort key (defaults to 'quoteVolume')
+        :param tickers: Tickers (from exchange.get_tickers()).
         :return: List of pairs
         """
 
-        tickers = self._freqtrade.exchange.get_tickers()
+        if self._pairlist_pos == 0:
+            # If VolumePairList is the first in the list, use fresh pairlist
             # check length so that we make sure that '/' is actually in the string
-        tickers = [v for k, v in tickers.items()
+            filtered_tickers = [v for k, v in tickers.items()
                                 if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency
                                     and v[key] is not None)]
-        sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key])
+        else:
+            # If other pairlist is in front, use the incomming pairlist.
+            filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
+
+        sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[key])
+
         # Validate whitelist to only have active market pairs
-        valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers])
+        pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
-        valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs]
+        pairs = self._verify_blacklist(pairs)
-
+        # Limit to X number of pairs
-        if self._freqtrade.strategy.stoploss is not None and self._precision_filter:
+        pairs = pairs[:self._number_pairs]
-
+        logger.info(f"Searching {self._number_pairs} pairs: {pairs}")
-            stop_prices = [self._freqtrade.get_target_bid(t["symbol"], t)
-                           * (1 - abs(self._freqtrade.strategy.stoploss)) for t in valid_tickers]
-            rates = [sp * 0.99 for sp in stop_prices]
-            logger.debug("\n".join([f"{sp} : {r}" for sp, r in zip(stop_prices[:10], rates[:10])]))
-            for i, t in enumerate(valid_tickers):
-                sp = self._freqtrade.exchange.symbol_price_prec(t["symbol"], stop_prices[i])
-                r = self._freqtrade.exchange.symbol_price_prec(t["symbol"], rates[i])
-                logger.debug(f"{t['symbol']} - {sp} : {r}")
-                if sp <= r:
-                    logger.info(f"Removed {t['symbol']} from whitelist, "
-                                f"because stop price {sp} would be <= stop limit {r}")
-                    valid_tickers.remove(t)
-
-        pairs = [s['symbol'] for s in valid_tickers]
-        logger.info(f"Searching pairs: {pairs[:self._number_pairs]}")
 
         return pairs

freqtrade/pairlist/pairlistmanager.py

@@ -0,0 +1,95 @@
+"""
+Static List provider
+
+Provides lists as configured in config.json
+
+ """
+from cachetools import TTLCache, cached
+import logging
+from typing import Dict, List
+
+from freqtrade import OperationalException
+from freqtrade.pairlist.IPairList import IPairList
+from freqtrade.resolvers import PairListResolver
+
+logger = logging.getLogger(__name__)
+
+
+class PairListManager():
+
+    def __init__(self, exchange, config: dict) -> None:
+        self._exchange = exchange
+        self._config = config
+        self._whitelist = self._config['exchange'].get('pair_whitelist')
+        self._blacklist = self._config['exchange'].get('pair_blacklist', [])
+        self._pairlists: List[IPairList] = []
+        self._tickers_needed = False
+        for pl in self._config.get('pairlists', None):
+            if 'method' not in pl:
+                logger.warning(f"No method in {pl}")
+                continue
+            pairl = PairListResolver(pl.get('method'),
+                                     exchange=exchange,
+                                     pairlistmanager=self,
+                                     config=config,
+                                     pairlistconfig=pl,
+                                     pairlist_pos=len(self._pairlists)
+                                     ).pairlist
+            self._tickers_needed = pairl.needstickers or self._tickers_needed
+            self._pairlists.append(pairl)
+
+        if not self._pairlists:
+            raise OperationalException("No Pairlist defined!")
+
+    @property
+    def whitelist(self) -> List[str]:
+        """
+        Has the current whitelist
+        """
+        return self._whitelist
+
+    @property
+    def blacklist(self) -> List[str]:
+        """
+        Has the current blacklist
+        -> no need to overwrite in subclasses
+        """
+        return self._blacklist
+
+    @property
+    def name_list(self) -> List[str]:
+        """
+        Get list of loaded pairlists names
+        """
+        return [p.name for p in self._pairlists]
+
+    def short_desc(self) -> List[Dict]:
+        """
+        List of short_desc for each pairlist
+        """
+        return [{p.name: p.short_desc()} for p in self._pairlists]
+
+    @cached(TTLCache(maxsize=1, ttl=1800))
+    def _get_cached_tickers(self):
+        return self._exchange.get_tickers()
+
+    def refresh_pairlist(self) -> None:
+        """
+        Run pairlist through all configured pairlists.
+        """
+
+        pairlist = self._whitelist.copy()
+
+        # tickers should be cached to avoid calling the exchange on each call.
+        tickers: Dict = {}
+        if self._tickers_needed:
+            tickers = self._get_cached_tickers()
+
+        # Process all pairlists in chain
+        for pl in self._pairlists:
+            pairlist = pl.filter_pairlist(pairlist, tickers)
+
+        # Validation against blacklist happens after the pairlists to ensure blacklist is respected.
+        pairlist = IPairList.verify_blacklist(pairlist, self.blacklist)
+
+        self._whitelist = pairlist

freqtrade/resolvers/iresolver.py

@@ -17,13 +17,13 @@ class IResolver:
     This class contains all the logic to load custom classes
     """
 
-    def build_search_paths(self, config, current_path: Path, user_subdir: str,
+    def build_search_paths(self, config, current_path: Path, user_subdir: Optional[str] = None,
                            extra_dir: Optional[str] = None) -> List[Path]:
 
-        abs_paths = [
+        abs_paths: List[Path] = [current_path]
-            config['user_data_dir'].joinpath(user_subdir),
+
-            current_path,
+        if user_subdir:
-        ]
+            abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir))
 
         if extra_dir:
             # Add extra directory to the top of the search paths

freqtrade/resolvers/pairlist_resolver.py

@@ -20,13 +20,18 @@ class PairListResolver(IResolver):
 
     __slots__ = ['pairlist']
 
-    def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None:
+    def __init__(self, pairlist_name: str, exchange, pairlistmanager,
+                 config: dict, pairlistconfig: dict, pairlist_pos: int) -> None:
         """
         Load the custom class from config parameter
         :param config: configuration dictionary or None
         """
-        self.pairlist = self._load_pairlist(pairlist_name, config, kwargs={'freqtrade': freqtrade,
+        self.pairlist = self._load_pairlist(pairlist_name, config,
-                                                                           'config': config})
+                                            kwargs={'exchange': exchange,
+                                                    'pairlistmanager': pairlistmanager,
+                                                    'config': config,
+                                                    'pairlistconfig': pairlistconfig,
+                                                    'pairlist_pos': pairlist_pos})
 
     def _load_pairlist(
             self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList:
@@ -40,7 +45,7 @@ class PairListResolver(IResolver):
         current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve()
 
         abs_paths = self.build_search_paths(config, current_path=current_path,
-                                            user_subdir='pairlist', extra_dir=None)
+                                            user_subdir=None, extra_dir=None)
 
         pairlist = self._load_object(paths=abs_paths, object_type=IPairList,
                                      object_name=pairlist_name, kwargs=kwargs)

freqtrade/rpc/rpc.py

@@ -462,7 +462,7 @@ class RPC:
 
     def _rpc_whitelist(self) -> Dict:
         """ Returns the currently active whitelist"""
-        res = {'method': self._freqtrade.pairlists.name,
+        res = {'method': self._freqtrade.pairlists.name_list,
                'length': len(self._freqtrade.active_pair_whitelist),
                'whitelist': self._freqtrade.active_pair_whitelist
                }
@@ -477,7 +477,7 @@ class RPC:
                         and pair not in self._freqtrade.pairlists.blacklist):
                     self._freqtrade.pairlists.blacklist.append(pair)
 
-        res = {'method': self._freqtrade.pairlists.name,
+        res = {'method': self._freqtrade.pairlists.name_list,
                'length': len(self._freqtrade.pairlists.blacklist),
                'blacklist': self._freqtrade.pairlists.blacklist,
                }

tests/conftest.py

@@ -242,6 +242,9 @@ def default_conf(testdatadir):
                 "HOT/BTC",
             ]
         },
+        "pairlists": [
+            {"method": "StaticPairList"}
+        ],
         "telegram": {
             "enabled": True,
             "token": "token",
@@ -573,6 +576,72 @@ def get_markets():
     }
 
 
+@pytest.fixture
+def shitcoinmarkets(markets):
+    """
+    Fixture with shitcoin markets - used to test filters in pairlists
+    """
+    shitmarkets = deepcopy(markets)
+    shitmarkets.update({'HOT/BTC': {
+            'id': 'HOTBTC',
+            'symbol': 'HOT/BTC',
+            'base': 'HOT',
+            'quote': 'BTC',
+            'active': True,
+            'precision': {
+                'base': 8,
+                'quote': 8,
+                'amount': 0,
+                'price': 8
+            },
+            'limits': {
+                'amount': {
+                    'min': 1.0,
+                    'max': 90000000.0
+                },
+                'price': {
+                    'min': None,
+                    'max': None
+                },
+                'cost': {
+                    'min': 0.001,
+                    'max': None
+                }
+            },
+            'info': {},
+        },
+        'FUEL/BTC': {
+            'id': 'FUELBTC',
+            'symbol': 'FUEL/BTC',
+            'base': 'FUEL',
+            'quote': 'BTC',
+            'active': True,
+            'precision': {
+                'base': 8,
+                'quote': 8,
+                'amount': 0,
+                'price': 8
+            },
+            'limits': {
+                'amount': {
+                    'min': 1.0,
+                    'max': 90000000.0
+                },
+                'price': {
+                    'min': 1e-08,
+                    'max': 1000.0
+                },
+                'cost': {
+                    'min': 0.001,
+                    'max': None
+                }
+            },
+            'info': {},
+        },
+        })
+    return shitmarkets
+
+
 @pytest.fixture
 def markets_empty():
     return MagicMock(return_value=[])
@@ -867,6 +936,50 @@ def tickers():
             'quoteVolume': 1215.14489611,
             'info': {}
         },
+        'HOT/BTC': {
+            'symbol': 'HOT/BTC',
+            'timestamp': 1572273518661,
+            'datetime': '2019-10-28T14:38:38.661Z',
+            'high': 0.00000011,
+            'low': 0.00000009,
+            'bid': 0.0000001,
+            'bidVolume': 1476027288.0,
+            'ask': 0.00000011,
+            'askVolume': 820153831.0,
+            'vwap': 0.0000001,
+            'open': 0.00000009,
+            'close': 0.00000011,
+            'last': 0.00000011,
+            'previousClose': 0.00000009,
+            'change': 0.00000002,
+            'percentage': 22.222,
+            'average': None,
+            'baseVolume': 1442290324.0,
+            'quoteVolume': 143.78311994,
+            'info': {}
+        },
+        'FUEL/BTC': {
+            'symbol': 'FUEL/BTC',
+            'timestamp': 1572340250771,
+            'datetime': '2019-10-29T09:10:50.771Z',
+            'high': 0.00000040,
+            'low': 0.00000035,
+            'bid': 0.00000036,
+            'bidVolume': 8932318.0,
+            'ask': 0.00000037,
+            'askVolume': 10140774.0,
+            'vwap': 0.00000037,
+            'open': 0.00000039,
+            'close': 0.00000037,
+            'last': 0.00000037,
+            'previousClose': 0.00000038,
+            'change': -0.00000002,
+            'percentage': -5.128,
+            'average': None,
+            'baseVolume': 168927742.0,
+            'quoteVolume': 62.68220262,
+            'info': {}
+        },
         'ETH/USDT': {
             'symbol': 'ETH/USDT',
             'timestamp': 1522014804118,

tests/pairlist/test_pairlist.py

@@ -2,11 +2,13 @@
 
 from unittest.mock import MagicMock, PropertyMock
 
+import pytest
+
 from freqtrade import OperationalException
 from freqtrade.constants import AVAILABLE_PAIRLISTS
 from freqtrade.resolvers import PairListResolver
-from tests.conftest import get_patched_freqtradebot
+from freqtrade.pairlist.pairlistmanager import PairListManager
-import pytest
+from tests.conftest import get_patched_freqtradebot, log_has_re
 
 # whitelist, blacklist
 
@@ -24,25 +26,39 @@ def whitelist_conf(default_conf):
     default_conf['exchange']['pair_blacklist'] = [
         'BLK/BTC'
     ]
-    default_conf['pairlist'] = {'method': 'StaticPairList',
+    default_conf['pairlists'] = [
-                                'config': {'number_assets': 3}
+        {
-                                }
+            "method": "VolumePairList",
-
+            "number_assets": 5,
+            "sort_key": "quoteVolume",
+        },
+    ]
     return default_conf
 
 
+@pytest.fixture(scope="function")
+def static_pl_conf(whitelist_conf):
+    whitelist_conf['pairlists'] = [
+        {
+            "method": "StaticPairList",
+        },
+    ]
+    return whitelist_conf
+
+
 def test_load_pairlist_noexist(mocker, markets, default_conf):
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    bot = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+    plm = PairListManager(bot.exchange, default_conf)
     with pytest.raises(OperationalException,
                        match=r"Impossible to load Pairlist 'NonexistingPairList'. "
                              r"This class does not exist or contains Python code errors."):
-        PairListResolver('NonexistingPairList', freqtradebot, default_conf).pairlist
+        PairListResolver('NonexistingPairList', bot.exchange, plm, default_conf, {}, 1)
 
 
-def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf):
+def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf):
 
-    freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
+    freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf)
 
     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
     freqtradebot.pairlists.refresh_pairlist()
@@ -51,50 +67,60 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf):
     # Ensure all except those in whitelist are removed
     assert set(whitelist) == set(freqtradebot.pairlists.whitelist)
     # Ensure config dict hasn't been changed
-    assert (whitelist_conf['exchange']['pair_whitelist'] ==
+    assert (static_pl_conf['exchange']['pair_whitelist'] ==
             freqtradebot.config['exchange']['pair_whitelist'])
 
 
-def test_refresh_pairlists(mocker, markets, whitelist_conf):
+def test_refresh_static_pairlist(mocker, markets, static_pl_conf):
-    freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
+    freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf)
-
+    mocker.patch.multiple(
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+        'freqtrade.exchange.Exchange',
+        exchange_has=MagicMock(return_value=True),
+        markets=PropertyMock(return_value=markets),
+    )
     freqtradebot.pairlists.refresh_pairlist()
     # List ordered by BaseVolume
     whitelist = ['ETH/BTC', 'TKN/BTC']
     # Ensure all except those in whitelist are removed
     assert set(whitelist) == set(freqtradebot.pairlists.whitelist)
-    assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist
+    assert static_pl_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist
 
 
-def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf):
+def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf):
-    whitelist_conf['pairlist'] = {'method': 'VolumePairList',
+
-                                  'config': {'number_assets': 5}
-                                  }
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        markets=PropertyMock(return_value=markets),
         get_tickers=tickers,
-        exchange_has=MagicMock(return_value=True)
+        exchange_has=MagicMock(return_value=True),
+    )
+    bot = get_patched_freqtradebot(mocker, whitelist_conf)
+    # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        markets=PropertyMock(return_value=shitcoinmarkets),
      )
-    freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
-
     # argument: use the whitelist dynamically by exchange-volume
-    whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']
+    whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']
-    freqtradebot.pairlists.refresh_pairlist()
+    bot.pairlists.refresh_pairlist()
 
-    assert whitelist == freqtradebot.pairlists.whitelist
+    assert whitelist == bot.pairlists.whitelist
 
-    whitelist_conf['pairlist'] = {'method': 'VolumePairList',
+    whitelist_conf['pairlists'] = [{'method': 'VolumePairList',
                                     'config': {}
                                     }
+                                   ]
+
     with pytest.raises(OperationalException,
                        match=r'`number_assets` not specified. Please check your configuration '
                              r'for "pairlist.config.number_assets"'):
-        PairListResolver('VolumePairList', freqtradebot, whitelist_conf).pairlist
+        PairListManager(bot.exchange, whitelist_conf)
 
 
 def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        exchange_has=MagicMock(return_value=True),
+    )
     freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_empty))
 
@@ -107,35 +133,75 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     assert set(whitelist) == set(pairslist)
 
 
-@pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [
+@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [
-    (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-    (False, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']),
+        "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']),
-    (False, "USDT", "quoteVolume", ['ETH/USDT']),
+    # Different sorting depending on quote or bid volume
-    (False, "ETH", "quoteVolume", []),  # this replaces tests that were removed from test_exchange
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
-    (True, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC"]),
+        "BTC",  ['HOT/BTC', 'FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']),
-    (True, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC"])
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
+        "USDT", ['ETH/USDT']),
+    # No pair for ETH ...
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
+     "ETH", []),
+    # Precisionfilter and quote volume
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']),
+    # Precisionfilter bid
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"},
+      {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']),
+    # PriceFilter and VolumePairList
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "PriceFilter", "low_price_ratio": 0.03}],
+        "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']),
+    # Hot is removed by precision_filter, Fuel by low_price_filter.
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "PrecisionFilter"},
+      {"method": "PriceFilter", "low_price_ratio": 0.02}
+      ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
+    # StaticPairlist Only
+    ([{"method": "StaticPairList"},
+      ], "BTC", ['ETH/BTC', 'TKN/BTC']),
+    # Static Pairlist before VolumePairList - sorting changes
+    ([{"method": "StaticPairList"},
+      {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"},
+      ], "BTC", ['TKN/BTC', 'ETH/BTC']),
 ])
-def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, base_currency, key,
+def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
-                                      whitelist_result, precision_filter) -> None:
+                                      pairlists, base_currency, whitelist_result,
-    whitelist_conf['pairlist']['method'] = 'VolumePairList'
+                                      caplog) -> None:
+    whitelist_conf['pairlists'] = pairlists
+
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
-    mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
-    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8))
 
-    freqtrade.pairlists._precision_filter = precision_filter
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          get_tickers=tickers,
+                          markets=PropertyMock(return_value=shitcoinmarkets),
+                          )
+
     freqtrade.config['stake_currency'] = base_currency
-    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key)
+    freqtrade.pairlists.refresh_pairlist()
-    assert sorted(whitelist) == sorted(whitelist_result)
+    whitelist = freqtrade.pairlists.whitelist
+
+    assert whitelist == whitelist_result
+    for pairlist in pairlists:
+        if pairlist['method'] == 'PrecisionFilter':
+            assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
+                              r'would be <= stop limit.*', caplog)
+        if pairlist['method'] == 'PriceFilter':
+            assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog)
 
 
 def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None:
-    default_conf['pairlist'] = {'method': 'VolumePairList',
+    default_conf['pairlists'] = [{'method': 'VolumePairList',
                                   'config': {'number_assets': 10}
-                                }
+                                  }]
-    mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
+
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False))
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          get_tickers=tickers,
+                          exchange_has=MagicMock(return_value=False),
+                          )
 
     with pytest.raises(OperationalException):
         get_patched_freqtradebot(mocker, default_conf)
@@ -143,13 +209,15 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None
 
 @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
 def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
-    whitelist_conf['pairlist']['method'] = pairlist
+    whitelist_conf['pairlists'][0]['method'] = pairlist
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True)
+                          )
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
 
-    assert freqtrade.pairlists.name == pairlist
+    assert freqtrade.pairlists.name_list == [pairlist]
-    assert pairlist in freqtrade.pairlists.short_desc()
+    assert pairlist in str(freqtrade.pairlists.short_desc())
     assert isinstance(freqtrade.pairlists.whitelist, list)
     assert isinstance(freqtrade.pairlists.blacklist, list)
 
@@ -157,20 +225,75 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
 @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
 @pytest.mark.parametrize("whitelist,log_message", [
     (['ETH/BTC', 'TKN/BTC'], ""),
-    (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"),  # TRX/ETH wrong stake
+    # TRX/ETH not in markets
-    (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"),  # BCH/BTC not available
+    (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"),
-    (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"),  # BLK/BTC in blacklist
+    # wrong stake
-    (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active")  # BTT/BTC is inactive
+    (['ETH/BTC', 'TKN/BTC', 'ETH/USDT'], "is not compatible with your stake currency"),
+    # BCH/BTC not available
+    (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"),
+    # BLK/BTC in blacklist
+    (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "in your blacklist. Removing "),
+    # BTT/BTC is inactive
+    (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active")
 ])
-def test_validate_whitelist(mocker, whitelist_conf, markets, pairlist, whitelist, caplog,
+def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog,
-                            log_message):
+                                       log_message, tickers):
-    whitelist_conf['pairlist']['method'] = pairlist
+    whitelist_conf['pairlists'][0]['method'] = pairlist
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
     caplog.clear()
 
-    new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist)
+    # Assign starting whitelist
+    new_whitelist = freqtrade.pairlists._pairlists[0]._whitelist_for_active_markets(whitelist)
 
     assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC'])
     assert log_message in caplog.text
+
+
+def test_volumepairlist_invalid_sortvalue(mocker, markets, whitelist_conf):
+    whitelist_conf['pairlists'][0].update({"sort_key": "asdf"})
+
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+    with pytest.raises(OperationalException,
+                       match=r"key asdf not in .*"):
+        get_patched_freqtradebot(mocker, whitelist_conf)
+
+
+def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+    bot = get_patched_freqtradebot(mocker, whitelist_conf)
+    assert bot.pairlists._pairlists[0]._last_refresh == 0
+    assert tickers.call_count == 0
+    bot.pairlists.refresh_pairlist()
+    assert tickers.call_count == 1
+
+    assert bot.pairlists._pairlists[0]._last_refresh != 0
+    lrf = bot.pairlists._pairlists[0]._last_refresh
+    bot.pairlists.refresh_pairlist()
+    assert tickers.call_count == 1
+    # Time should not be updated.
+    assert bot.pairlists._pairlists[0]._last_refresh == lrf
+
+
+def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog):
+    del whitelist_conf['pairlists'][0]['method']
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+    with pytest.raises(OperationalException,
+                       match=r"No Pairlist defined!"):
+        get_patched_freqtradebot(mocker, whitelist_conf)
+    assert log_has_re("No method in .*", caplog)
+
+    whitelist_conf['pairlists'] = []
+
+    with pytest.raises(OperationalException,
+                       match=r"No Pairlist defined!"):
+        get_patched_freqtradebot(mocker, whitelist_conf)

tests/rpc/test_rpc.py

@@ -736,21 +736,23 @@ def test_rpc_whitelist(mocker, default_conf) -> None:
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
     rpc = RPC(freqtradebot)
     ret = rpc._rpc_whitelist()
-    assert ret['method'] == 'StaticPairList'
+    assert len(ret['method']) == 1
+    assert 'StaticPairList' in ret['method']
     assert ret['whitelist'] == default_conf['exchange']['pair_whitelist']
 
 
 def test_rpc_whitelist_dynamic(mocker, default_conf) -> None:
-    default_conf['pairlist'] = {'method': 'VolumePairList',
+    default_conf['pairlists'] = [{'method': 'VolumePairList',
-                                'config': {'number_assets': 4}
+                                  'number_assets': 4,
-                                }
+                                  }]
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
     mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
 
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
     rpc = RPC(freqtradebot)
     ret = rpc._rpc_whitelist()
-    assert ret['method'] == 'VolumePairList'
+    assert len(ret['method']) == 1
+    assert 'VolumePairList' in ret['method']
     assert ret['length'] == 4
     assert ret['whitelist'] == default_conf['exchange']['pair_whitelist']
 
@@ -761,13 +763,14 @@ def test_rpc_blacklist(mocker, default_conf) -> None:
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
     rpc = RPC(freqtradebot)
     ret = rpc._rpc_blacklist(None)
-    assert ret['method'] == 'StaticPairList'
+    assert len(ret['method']) == 1
+    assert 'StaticPairList' in ret['method']
     assert len(ret['blacklist']) == 2
     assert ret['blacklist'] == default_conf['exchange']['pair_blacklist']
     assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC']
 
     ret = rpc._rpc_blacklist(["ETH/BTC"])
-    assert ret['method'] == 'StaticPairList'
+    assert 'StaticPairList' in ret['method']
     assert len(ret['blacklist']) == 3
     assert ret['blacklist'] == default_conf['exchange']['pair_blacklist']
     assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC']

tests/rpc/test_rpc_apiserver.py

@@ -460,7 +460,7 @@ def test_api_blacklist(botclient, mocker):
     assert_response(rc)
     assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"],
                        "length": 2,
-                       "method": "StaticPairList"}
+                       "method": ["StaticPairList"]}
 
     # Add ETH/BTC to blacklist
     rc = client_post(client, f"{BASE_URI}/blacklist",
@@ -468,7 +468,7 @@ def test_api_blacklist(botclient, mocker):
     assert_response(rc)
     assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"],
                        "length": 3,
-                       "method": "StaticPairList"}
+                       "method": ["StaticPairList"]}
 
 
 def test_api_whitelist(botclient):
@@ -478,7 +478,7 @@ def test_api_whitelist(botclient):
     assert_response(rc)
     assert rc.json == {"whitelist": ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'],
                        "length": 4,
-                       "method": "StaticPairList"}
+                       "method": ["StaticPairList"]}
 
 
 def test_api_forcebuy(botclient, mocker, fee):

tests/rpc/test_rpc_telegram.py

@@ -1050,8 +1050,8 @@ def test_whitelist_static(default_conf, update, mocker) -> None:
 
     telegram._whitelist(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
-    assert ('Using whitelist `StaticPairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`'
+    assert ("Using whitelist `['StaticPairList']` with 4 pairs\n"
-            in msg_mock.call_args_list[0][0][0])
+            "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0])
 
 
 def test_whitelist_dynamic(default_conf, update, mocker) -> None:
@@ -1062,17 +1062,17 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None:
         _send_msg=msg_mock
     )
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
-    default_conf['pairlist'] = {'method': 'VolumePairList',
+    default_conf['pairlists'] = [{'method': 'VolumePairList',
-                                'config': {'number_assets': 4}
+                                 'number_assets': 4
-                                }
+                                  }]
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
 
     telegram = Telegram(freqtradebot)
 
     telegram._whitelist(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
-    assert ('Using whitelist `VolumePairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`'
+    assert ("Using whitelist `['VolumePairList']` with 4 pairs\n"
-            in msg_mock.call_args_list[0][0][0])
+            "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0])
 
 
 def test_blacklist_static(default_conf, update, mocker) -> None:

tests/test_configuration.py

@@ -777,9 +777,9 @@ def test_validate_whitelist(default_conf):
 
     conf = deepcopy(default_conf)
 
-    conf.update({"pairlist": {
+    conf.update({"pairlists": [{
         "method": "VolumePairList",
-    }})
+    }]})
     # Dynamic whitelist should not care about pair_whitelist
     validate_config_consistency(conf)
     del conf['exchange']['pair_whitelist']
@@ -997,6 +997,18 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca
     assert default_conf[setting[0]][setting[1]] == setting[5]
 
 
+def test_process_deprecated_setting_pairlists(mocker, default_conf, caplog):
+    patched_configuration_load_config_file(mocker, default_conf)
+    default_conf.update({'pairlist': {
+        'method': 'VolumePairList',
+        'config': {'precision_filter': True}
+    }})
+
+    process_temporary_deprecated_settings(default_conf)
+    assert log_has_re(r'DEPRECATED.*precision_filter.*', caplog)
+    assert log_has_re(r'DEPRECATED.*in pairlist is deprecated and must be moved*', caplog)
+
+
 def test_check_conflicting_settings(mocker, default_conf, caplog):
     patched_configuration_load_config_file(mocker, default_conf)