stable/tests/plugins/test_pairlist.py

# pragma pylint: disable=missing-docstring,C0103,protected-access

import logging
import time
from copy import deepcopy
from datetime import timedelta
from unittest.mock import MagicMock, PropertyMock

import pandas as pd
import pytest
import time_machine

from freqtrade.constants import AVAILABLE_PAIRLISTS
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import CandleType, RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.persistence import Trade
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
from freqtrade.plugins.pairlistmanager import PairListManager
from freqtrade.resolvers import PairListResolver
from tests.conftest import (EXMS, create_mock_trades_usdt, get_patched_exchange,
                            get_patched_freqtradebot, log_has, log_has_re, num_log_has)


# Exclude RemotePairList from tests.
# It has a mandatory parameter, and requires special handling, which happens in test_remotepairlist.
TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ['RemotePairList']]


@pytest.fixture(scope="function")
def whitelist_conf(default_conf):
    default_conf['stake_currency'] = 'BTC'
    default_conf['exchange']['pair_whitelist'] = [
        'ETH/BTC',
        'TKN/BTC',
        'TRST/BTC',
        'SWT/BTC',
        'BCC/BTC',
        'HOT/BTC',
    ]
    default_conf['exchange']['pair_blacklist'] = [
        'BLK/BTC'
    ]
    default_conf['pairlists'] = [
        {
            "method": "VolumePairList",
            "number_assets": 5,
            "sort_key": "quoteVolume",
        },
    ]
    default_conf.update({
        "external_message_consumer": {
            "enabled": True,
            "producers": [],
        }
    })
    return default_conf


@pytest.fixture(scope="function")
def whitelist_conf_2(default_conf):
    default_conf['stake_currency'] = 'BTC'
    default_conf['exchange']['pair_whitelist'] = [
        'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC',
        'BTT/BTC', 'HOT/BTC', 'FUEL/BTC', 'XRP/BTC'
    ]
    default_conf['exchange']['pair_blacklist'] = [
        'BLK/BTC'
    ]
    default_conf['pairlists'] = [
        # {   "method": "StaticPairList"},
        {
            "method": "VolumePairList",
            "number_assets": 5,
            "sort_key": "quoteVolume",
            "refresh_period": 0,
        },
    ]
    return default_conf


@pytest.fixture(scope="function")
def whitelist_conf_agefilter(default_conf):
    default_conf['stake_currency'] = 'BTC'
    default_conf['exchange']['pair_whitelist'] = [
        'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC',
        'BTT/BTC', 'HOT/BTC', 'FUEL/BTC', 'XRP/BTC'
    ]
    default_conf['exchange']['pair_blacklist'] = [
        'BLK/BTC'
    ]
    default_conf['pairlists'] = [
        {
            "method": "VolumePairList",
            "number_assets": 5,
            "sort_key": "quoteVolume",
            "refresh_period": -1,
        },
        {
            "method": "AgeFilter",
            "min_days_listed": 2,
            "max_days_listed": 100
        }
    ]
    return default_conf


@pytest.fixture(scope="function")
def static_pl_conf(whitelist_conf):
    whitelist_conf['pairlists'] = [
        {
            "method": "StaticPairList",
        },
    ]
    return whitelist_conf


def test_log_cached(mocker, static_pl_conf, markets, tickers):
    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )
    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
    logmock = MagicMock()
    # Assign starting whitelist
    pl = freqtrade.pairlists._pairlist_handlers[0]
    pl.log_once('Hello world', logmock)
    assert logmock.call_count == 1
    pl.log_once('Hello world', logmock)
    assert logmock.call_count == 1
    assert pl._log_cache.currsize == 1
    assert ('Hello world',) in pl._log_cache._Cache__data

    pl.log_once('Hello world2', logmock)
    assert logmock.call_count == 2
    assert pl._log_cache.currsize == 2


def test_load_pairlist_noexist(mocker, markets, default_conf):
    freqtrade = get_patched_freqtradebot(mocker, default_conf)
    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets))
    plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
    with pytest.raises(OperationalException,
                       match=r"Impossible to load Pairlist 'NonexistingPairList'. "
                             r"This class does not exist or contains Python code errors."):
        PairListResolver.load_pairlist('NonexistingPairList', freqtrade.exchange, plm,
                                       default_conf, {}, 1)


def test_load_pairlist_verify_multi(mocker, markets_static, default_conf):
    freqtrade = get_patched_freqtradebot(mocker, default_conf)
    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets_static))
    plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
    # Call different versions one after the other, should always consider what was passed in
    # and have no side-effects (therefore the same check multiple times)
    assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', ], print) == ['ETH/BTC', 'XRP/BTC']
    assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', 'BUUU/BTC'], print) == ['ETH/BTC', 'XRP/BTC']
    assert plm.verify_whitelist(['XRP/BTC', 'BUUU/BTC'], print) == ['XRP/BTC']
    assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', ], print) == ['ETH/BTC', 'XRP/BTC']
    assert plm.verify_whitelist(['ETH/USDT', 'XRP/USDT', ], print) == ['ETH/USDT', ]
    assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', ], print) == ['ETH/BTC', 'XRP/BTC']


def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf):

    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)

    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets))
    freqtrade.pairlists.refresh_pairlist()
    # List ordered by BaseVolume
    whitelist = ['ETH/BTC', 'TKN/BTC']
    # Ensure all except those in whitelist are removed
    assert set(whitelist) == set(freqtrade.pairlists.whitelist)
    # Ensure config dict hasn't been changed
    assert (static_pl_conf['exchange']['pair_whitelist'] ==
            freqtrade.config['exchange']['pair_whitelist'])


def test_refresh_static_pairlist(mocker, markets, static_pl_conf):
    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
    mocker.patch.multiple(
        EXMS,
        exchange_has=MagicMock(return_value=True),
        markets=PropertyMock(return_value=markets),
    )
    freqtrade.pairlists.refresh_pairlist()
    # List ordered by BaseVolume
    whitelist = ['ETH/BTC', 'TKN/BTC']
    # Ensure all except those in whitelist are removed
    assert set(whitelist) == set(freqtrade.pairlists.whitelist)
    assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist


@pytest.mark.parametrize('pairs,expected', [
    (['NOEXIST/BTC', r'\+WHAT/BTC'],
     ['ETH/BTC', 'TKN/BTC', 'TRST/BTC', 'NOEXIST/BTC', 'SWT/BTC', 'BCC/BTC', 'HOT/BTC']),
    (['NOEXIST/BTC', r'*/BTC'],  # This is an invalid regex
     []),
])
def test_refresh_static_pairlist_noexist(mocker, markets, static_pl_conf, pairs, expected, caplog):

    static_pl_conf['pairlists'][0]['allow_inactive'] = True
    static_pl_conf['exchange']['pair_whitelist'] += pairs
    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
    mocker.patch.multiple(
        EXMS,
        exchange_has=MagicMock(return_value=True),
        markets=PropertyMock(return_value=markets),
    )
    freqtrade.pairlists.refresh_pairlist()

    # Ensure all except those in whitelist are removed
    assert set(expected) == set(freqtrade.pairlists.whitelist)
    assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
    if not expected:
        assert log_has_re(r'Pair whitelist contains an invalid Wildcard: Wildcard error.*', caplog)


def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog):
    static_pl_conf['exchange']['pair_blacklist'] = ['*/BTC']
    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
    mocker.patch.multiple(
        EXMS,
        exchange_has=MagicMock(return_value=True),
        markets=PropertyMock(return_value=markets),
    )
    freqtrade.pairlists.refresh_pairlist()
    whitelist = []
    # Ensure all except those in whitelist are removed
    assert set(whitelist) == set(freqtrade.pairlists.whitelist)
    assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
    log_has_re(r"Pair blacklist contains an invalid Wildcard.*", caplog)


def test_remove_logs_for_pairs_already_in_blacklist(mocker, markets, static_pl_conf, caplog):
    logger = logging.getLogger(__name__)
    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
    mocker.patch.multiple(
        EXMS,
        exchange_has=MagicMock(return_value=True),
        markets=PropertyMock(return_value=markets),
    )
    freqtrade.pairlists.refresh_pairlist()
    whitelist = ['ETH/BTC', 'TKN/BTC']
    caplog.clear()
    caplog.set_level(logging.INFO)

    # Ensure all except those in whitelist are removed.
    assert set(whitelist) == set(freqtrade.pairlists.whitelist)
    assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
    # Ensure that log message wasn't generated.
    assert not log_has('Pair BLK/BTC in your blacklist. Removing it from whitelist...', caplog)

    for _ in range(3):
        new_whitelist = freqtrade.pairlists.verify_blacklist(
            whitelist + ['BLK/BTC'], logger.warning)
        # Ensure that the pair is removed from the white list, and properly logged.
        assert set(whitelist) == set(new_whitelist)
    assert num_log_has('Pair BLK/BTC in your blacklist. Removing it from whitelist...',
                       caplog) == 1


def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf):

    mocker.patch.multiple(
        EXMS,
        get_tickers=tickers,
        exchange_has=MagicMock(return_value=True),
    )
    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture
    mocker.patch.multiple(
        EXMS,
        markets=PropertyMock(return_value=shitcoinmarkets),
    )
    # argument: use the whitelist dynamically by exchange-volume
    whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']
    freqtrade.pairlists.refresh_pairlist()
    assert whitelist == freqtrade.pairlists.whitelist

    whitelist_conf['pairlists'] = [{'method': 'VolumePairList'}]
    with pytest.raises(OperationalException,
                       match=r'`number_assets` not specified. Please check your configuration '
                             r'for "pairlist.config.number_assets"'):
        PairListManager(freqtrade.exchange, whitelist_conf, MagicMock())


def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_conf_2):

    tickers_dict = tickers()

    mocker.patch.multiple(
        EXMS,
        exchange_has=MagicMock(return_value=True),
    )
    # Remove caching of ticker data to emulate changing volume by the time of second call
    mocker.patch.multiple(
        'freqtrade.plugins.pairlistmanager.PairListManager',
        _get_cached_tickers=MagicMock(return_value=tickers_dict),
    )
    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_2)
    # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture
    mocker.patch.multiple(
        EXMS,
        markets=PropertyMock(return_value=shitcoinmarkets),
    )

    whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']
    freqtrade.pairlists.refresh_pairlist()
    assert whitelist == freqtrade.pairlists.whitelist

    # Delay to allow 0 TTL cache to expire...
    time.sleep(1)
    whitelist = ['FUEL/BTC', 'ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']
    tickers_dict['FUEL/BTC']['quoteVolume'] = 10000.0
    freqtrade.pairlists.refresh_pairlist()
    assert whitelist == freqtrade.pairlists.whitelist


def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
    mocker.patch.multiple(
        EXMS,
        exchange_has=MagicMock(return_value=True),
    )
    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets_empty))

    # argument: use the whitelist dynamically by exchange-volume
    whitelist = []
    whitelist_conf['exchange']['pair_whitelist'] = []
    freqtrade.pairlists.refresh_pairlist()
    pairslist = whitelist_conf['exchange']['pair_whitelist']

    assert set(whitelist) == set(pairslist)


@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [
    # VolumePairList only
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']),
    # No pair for ETH, VolumePairList
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
     "ETH", []),
    # No pair for ETH, StaticPairList
    ([{"method": "StaticPairList"}],
     "ETH", []),
    # No pair for ETH, all handlers
    ([{"method": "StaticPairList"},
      {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": None},
      {"method": "PrecisionFilter"},
      {"method": "PriceFilter", "low_price_ratio": 0.03},
      {"method": "SpreadFilter", "max_spread_ratio": 0.005},
      {"method": "ShuffleFilter"}, {"method": "PerformanceFilter"}],
     "ETH", []),
    # AgeFilter and VolumePairList (require 2 days only, all should pass age test)
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": 100}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
    # AgeFilter and VolumePairList (require 10 days, all should fail age test)
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": None}],
     "BTC", []),
    # AgeFilter and VolumePairList (all pair listed > 2, all should fail age test)
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}],
     "BTC", []),
    # AgeFilter and VolumePairList LTC/BTC has 6 candles - removes all
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 5}],
     "BTC", []),
    # AgeFilter and VolumePairList LTC/BTC has 6 candles - passes
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 10}],
     "BTC", ["LTC/BTC"]),
    # Precisionfilter and quote volume
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "PrecisionFilter"}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "PrecisionFilter"}],
     "USDT", ['ETH/USDT', 'NANO/USDT']),
    # PriceFilter and VolumePairList
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "PriceFilter", "low_price_ratio": 0.03}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
    # PriceFilter and VolumePairList
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "PriceFilter", "low_price_ratio": 0.03}],
     "USDT", ['ETH/USDT', 'NANO/USDT']),
    # Hot is removed by precision_filter, Fuel by low_price_ratio, Ripple by min_price.
    ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"},
      {"method": "PrecisionFilter"},
      {"method": "PriceFilter", "low_price_ratio": 0.02, "min_price": 0.01}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
    # Hot is removed by precision_filter, Fuel by low_price_ratio, Ethereum by max_price.
    ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"},
      {"method": "PrecisionFilter"},
      {"method": "PriceFilter", "low_price_ratio": 0.02, "max_price": 0.05}],
     "BTC", ['TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
    # HOT and XRP are removed because below 1250 quoteVolume
    ([{"method": "VolumePairList", "number_assets": 5,
       "sort_key": "quoteVolume", "min_value": 1250}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
    # StaticPairlist only
    ([{"method": "StaticPairList"}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
    # Static Pairlist before VolumePairList - sorting changes
    # SpreadFilter
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "SpreadFilter", "max_spread_ratio": 0.005}],
     "USDT", ['ETH/USDT']),
    # ShuffleFilter
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "ShuffleFilter", "seed": 77}],
     "USDT", ['ADADOUBLE/USDT', 'ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']),
    # ShuffleFilter, other seed
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "ShuffleFilter", "seed": 42}],
     "USDT", ['ADAHALF/USDT', 'NANO/USDT', 'ADADOUBLE/USDT', 'ETH/USDT']),
    # ShuffleFilter, no seed
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "ShuffleFilter"}],
     "USDT", 3),  # whitelist_result is integer -- check only length of randomized pairlist
    # AgeFilter only
    ([{"method": "AgeFilter", "min_days_listed": 2}],
     "BTC", 'filter_at_the_beginning'),  # OperationalException expected
    # PrecisionFilter after StaticPairList
    ([{"method": "StaticPairList"},
      {"method": "PrecisionFilter"}],
     "BTC", ['ETH/BTC', 'TKN/BTC']),
    # PrecisionFilter only
    ([{"method": "PrecisionFilter"}],
     "BTC", 'filter_at_the_beginning'),  # OperationalException expected
    # PriceFilter after StaticPairList
    ([{"method": "StaticPairList"},
      {"method": "PriceFilter", "low_price_ratio": 0.02, "min_price": 0.000001, "max_price": 0.1}],
     "BTC", ['ETH/BTC', 'TKN/BTC']),
    # PriceFilter only
    ([{"method": "PriceFilter", "low_price_ratio": 0.02}],
     "BTC", 'filter_at_the_beginning'),  # OperationalException expected
    # ShuffleFilter after StaticPairList
    ([{"method": "StaticPairList"},
      {"method": "ShuffleFilter", "seed": 42}],
     "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC']),
    # ShuffleFilter only
    ([{"method": "ShuffleFilter", "seed": 42}],
     "BTC", 'filter_at_the_beginning'),  # OperationalException expected
    # PerformanceFilter after StaticPairList
    ([{"method": "StaticPairList"},
      {"method": "PerformanceFilter"}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
    # PerformanceFilter only
    ([{"method": "PerformanceFilter"}],
     "BTC", 'filter_at_the_beginning'),  # OperationalException expected
    # SpreadFilter after StaticPairList
    ([{"method": "StaticPairList"},
      {"method": "SpreadFilter", "max_spread_ratio": 0.005}],
     "BTC", ['ETH/BTC', 'TKN/BTC']),
    # SpreadFilter only
    ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}],
     "BTC", 'filter_at_the_beginning'),  # OperationalException expected
    # Static Pairlist after VolumePairList, on a non-first position (appends pairs)
    ([{"method": "VolumePairList", "number_assets": 2, "sort_key": "quoteVolume"},
      {"method": "StaticPairList"}],
        "BTC", ['ETH/BTC', 'TKN/BTC', 'TRST/BTC', 'SWT/BTC', 'BCC/BTC', 'HOT/BTC']),
    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
      {"method": "PriceFilter", "low_price_ratio": 0.02}],
        "USDT", ['ETH/USDT', 'NANO/USDT']),
    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
      {"method": "PriceFilter", "max_value": 0.000001}],
        "USDT", ['NANO/USDT']),
    ([{"method": "StaticPairList"},
      {"method": "RangeStabilityFilter", "lookback_days": 10,
       "min_rate_of_change": 0.01, "refresh_period": 1440}],
     "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
    ([{"method": "StaticPairList"},
      {"method": "RangeStabilityFilter", "lookback_days": 10,
       "max_rate_of_change": 0.01, "refresh_period": 1440}],
     "BTC", []),  # All removed because of max_rate_of_change being 0.017
    ([{"method": "StaticPairList"},
      {"method": "RangeStabilityFilter", "lookback_days": 10,
        "min_rate_of_change": 0.018, "max_rate_of_change": 0.02, "refresh_period": 1440}],
     "BTC", []),  # All removed - limits are above the highest change_rate
    ([{"method": "StaticPairList"},
      {"method": "VolatilityFilter", "lookback_days": 3,
       "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}],
     "BTC", ['ETH/BTC', 'TKN/BTC']),
    # VolumePairList with no offset = unchanged pairlist
    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
      {"method": "OffsetFilter", "offset": 0, "number_assets": 0}],
     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']),
    # VolumePairList with offset = 2
    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
      {"method": "OffsetFilter", "offset": 2}],
     "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']),
    # VolumePairList with offset and limit
    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
      {"method": "OffsetFilter", "offset": 1, "number_assets": 2}],
     "USDT", ['NANO/USDT', 'ADAHALF/USDT']),
    # VolumePairList with higher offset, than total pairlist
    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
      {"method": "OffsetFilter", "offset": 100}],
     "USDT", [])
])
def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                      ohlcv_history, pairlists, base_currency,
                                      whitelist_result, caplog) -> None:
    whitelist_conf['pairlists'] = pairlists
    whitelist_conf['stake_currency'] = base_currency

    ohlcv_history_high_vola = ohlcv_history.copy()
    ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090

    ohlcv_data = {
        ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('LTC/BTC', '1d', CandleType.SPOT): pd.concat([ohlcv_history, ohlcv_history]),
        ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('HOT/BTC', '1d', CandleType.SPOT): ohlcv_history_high_vola,
    }

    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))

    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    mocker.patch.multiple(EXMS,
                          get_tickers=tickers,
                          markets=PropertyMock(return_value=shitcoinmarkets)
                          )
    mocker.patch.multiple(
        EXMS,
        refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
    )

    # Provide for PerformanceFilter's dependency
    mocker.patch.multiple('freqtrade.persistence.Trade',
                          get_overall_performance=MagicMock(return_value=[])
                          )

    # Set whitelist_result to None if pairlist is invalid and should produce exception
    if whitelist_result == 'filter_at_the_beginning':
        with pytest.raises(OperationalException,
                           match=r"This Pairlist Handler should not be used at the first position "
                                 r"in the list of Pairlist Handlers."):
            freqtrade.pairlists.refresh_pairlist()
    else:
        freqtrade.pairlists.refresh_pairlist()
        whitelist = freqtrade.pairlists.whitelist

        assert isinstance(whitelist, list)

        # Verify length of pairlist matches (used for ShuffleFilter without seed)
        if type(whitelist_result) is list:
            assert whitelist == whitelist_result
        else:
            len(whitelist) == whitelist_result

        for pairlist in pairlists:
            if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
                    len(ohlcv_history) < pairlist['min_days_listed']:
                assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
                                  r'.* day.*', caplog)
            if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \
                    len(ohlcv_history) > pairlist['max_days_listed']:
                assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
                                  r'.* day.* or more than .* day', caplog)
            if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
                assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
                                  r'would be <= stop limit.*', caplog)
            if pairlist['method'] == 'PriceFilter' and whitelist_result:
                assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or
                        log_has_re(r'^Removed .* from whitelist, '
                                   r'because last price < .*%$', caplog) or
                        log_has_re(r'^Removed .* from whitelist, '
                                   r'because last price > .*%$', caplog) or
                        log_has_re(r'^Removed .* from whitelist, '
                                   r'because min value change of .*', caplog) or
                        log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] "
                                   r"is empty.*", caplog))
            if pairlist['method'] == 'VolumePairList':
                logmsg = ("DEPRECATED: using any key other than quoteVolume for "
                          "VolumePairList is deprecated.")
                if pairlist['sort_key'] != 'quoteVolume':
                    assert log_has(logmsg, caplog)
                else:
                    assert not log_has(logmsg, caplog)
            if pairlist["method"] == 'VolatilityFilter':
                assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog)


@pytest.mark.parametrize("pairlists,base_currency,exchange,volumefilter_result", [
    # default refresh of 1800 to small for daily candle lookback
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
       "lookback_days": 1}],
     "BTC", "binance", "default_refresh_too_short"),  # OperationalException expected
    # ambigous configuration with lookback days and period
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
       "lookback_days": 1, "lookback_period": 1}],
     "BTC", "binance", "lookback_days_and_period"),  # OperationalException expected
    # negative lookback period
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
       "lookback_timeframe": "1d", "lookback_period": -1}],
     "BTC", "binance", "lookback_period_negative"),  # OperationalException expected
    # lookback range exceedes exchange limit
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
       "lookback_timeframe": "1m", "lookback_period": 2000, "refresh_period": 3600}],
     "BTC", "binance", "lookback_exceeds_exchange_request_size"),  # OperationalException expected
    # expecing pairs as given
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
       "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
     "BTC", "binance", ['LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC', 'HOT/BTC']),
    # expecting pairs as input, because 1h candles are not available
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
       "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}],
     "BTC", "binance", ['ETH/BTC', 'LTC/BTC', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']),
    # ftx data is already in Quote currency, therefore won't require conversion
    # ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
    #    "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
    #  "BTC", "ftx", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']),
])
def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history,
                              pairlists, base_currency, exchange, volumefilter_result) -> None:
    whitelist_conf['pairlists'] = pairlists
    whitelist_conf['stake_currency'] = base_currency
    whitelist_conf['exchange']['name'] = exchange

    ohlcv_history_high_vola = ohlcv_history.copy()
    ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090

    # create candles for medium overall volume with last candle high volume
    ohlcv_history_medium_volume = ohlcv_history.copy()
    ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5

    # create candles for high volume with all candles high volume, but very low price.
    ohlcv_history_high_volume = ohlcv_history.copy()
    ohlcv_history_high_volume['volume'] = 10
    ohlcv_history_high_volume['low'] = ohlcv_history_high_volume.loc[:, 'low'] * 0.01
    ohlcv_history_high_volume['high'] = ohlcv_history_high_volume.loc[:, 'high'] * 0.01
    ohlcv_history_high_volume['close'] = ohlcv_history_high_volume.loc[:, 'close'] * 0.01

    ohlcv_data = {
        ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history_medium_volume,
        ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history_high_vola,
        ('HOT/BTC', '1d', CandleType.SPOT): ohlcv_history_high_volume,
    }

    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))

    if volumefilter_result == 'default_refresh_too_short':
        with pytest.raises(OperationalException,
                           match=r'Refresh period of [0-9]+ seconds is smaller than one timeframe '
                                 r'of [0-9]+.*\. Please adjust refresh_period to at least [0-9]+ '
                                 r'and restart the bot\.'):
            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
        return
    elif volumefilter_result == 'lookback_days_and_period':
        with pytest.raises(OperationalException,
                           match=r'Ambigous configuration: lookback_days and lookback_period both '
                                 r'set in pairlist config\..*'):
            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    elif volumefilter_result == 'lookback_period_negative':
        with pytest.raises(OperationalException,
                           match=r'VolumeFilter requires lookback_period to be >= 0'):
            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    elif volumefilter_result == 'lookback_exceeds_exchange_request_size':
        with pytest.raises(OperationalException,
                           match=r'VolumeFilter requires lookback_period to not exceed '
                           r'exchange max request size \([0-9]+\)'):
            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    else:
        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
        mocker.patch.multiple(
            EXMS,
            get_tickers=tickers,
            markets=PropertyMock(return_value=shitcoinmarkets)
        )

        # remove ohlcv when looback_timeframe != 1d
        # to enforce fallback to ticker data
        if 'lookback_timeframe' in pairlists[0]:
            if pairlists[0]['lookback_timeframe'] != '1d':
                ohlcv_data = []

        mocker.patch.multiple(
            EXMS,
            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
        )

        freqtrade.pairlists.refresh_pairlist()
        whitelist = freqtrade.pairlists.whitelist

        assert isinstance(whitelist, list)
        assert whitelist == volumefilter_result


def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
    whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}]
    del whitelist_conf['stoploss']

    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))

    with pytest.raises(OperationalException,
                       match=r"PrecisionFilter can only work with stoploss defined\..*"):
        PairListManager(MagicMock, whitelist_conf, MagicMock())


def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
    whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}]
    if hasattr(Trade, 'session'):
        del Trade.session
    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))
    exchange = get_patched_exchange(mocker, whitelist_conf)
    pm = PairListManager(exchange, whitelist_conf, MagicMock())
    pm.refresh_pairlist()

    assert log_has("PerformanceFilter is not available in this mode.", caplog)


def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None:
    whitelist_conf['pairlists'] = [
        {"method": "StaticPairList"},
        {"method": "ShuffleFilter", "seed": 43}
    ]

    exchange = get_patched_exchange(mocker, whitelist_conf)
    plm = PairListManager(exchange, whitelist_conf)
    assert log_has("Backtesting mode detected, applying seed value: 43", caplog)

    with time_machine.travel("2021-09-01 05:01:00 +00:00") as t:
        plm.refresh_pairlist()
        pl1 = deepcopy(plm.whitelist)
        plm.refresh_pairlist()
        assert plm.whitelist == pl1

        t.shift(timedelta(minutes=10))
        plm.refresh_pairlist()
        assert plm.whitelist != pl1

    caplog.clear()
    whitelist_conf['runmode'] = RunMode.DRY_RUN
    plm = PairListManager(exchange, whitelist_conf)
    assert not log_has("Backtesting mode detected, applying seed value: 42", caplog)
    assert log_has("Live mode detected, not applying seed.", caplog)


@pytest.mark.usefixtures("init_persistence")
def test_PerformanceFilter_lookback(mocker, default_conf_usdt, fee, caplog) -> None:
    default_conf_usdt['exchange']['pair_whitelist'].extend(['ADA/USDT', 'XRP/USDT', 'ETC/USDT'])
    default_conf_usdt['pairlists'] = [
        {"method": "StaticPairList"},
        {"method": "PerformanceFilter", "minutes": 60, "min_profit": 0.01}
    ]
    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))
    exchange = get_patched_exchange(mocker, default_conf_usdt)
    pm = PairListManager(exchange, default_conf_usdt)
    pm.refresh_pairlist()

    assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']

    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
        create_mock_trades_usdt(fee)
        pm.refresh_pairlist()
        assert pm.whitelist == ['XRP/USDT', 'NEO/USDT']
        assert log_has_re(r'Removing pair .* since .* is below .*', caplog)

        # Move to "outside" of lookback window, so original sorting is restored.
        t.move_to("2021-09-01 07:00:00 +00:00")
        pm.refresh_pairlist()
        assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']


@pytest.mark.usefixtures("init_persistence")
def test_PerformanceFilter_keep_mid_order(mocker, default_conf_usdt, fee, caplog) -> None:
    default_conf_usdt['exchange']['pair_whitelist'].extend(['ADA/USDT', 'ETC/USDT'])
    default_conf_usdt['pairlists'] = [
        {"method": "StaticPairList", "allow_inactive": True},
        {"method": "PerformanceFilter", "minutes": 60, }
    ]
    mocker.patch(f'{EXMS}.exchange_has', return_value=True)
    exchange = get_patched_exchange(mocker, default_conf_usdt)
    pm = PairListManager(exchange, default_conf_usdt)
    pm.refresh_pairlist()

    assert pm.whitelist == ['ETH/USDT', 'LTC/USDT', 'XRP/USDT',
                            'NEO/USDT', 'TKN/USDT', 'ADA/USDT', 'ETC/USDT']

    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
        create_mock_trades_usdt(fee)
        pm.refresh_pairlist()
        assert pm.whitelist == ['XRP/USDT', 'NEO/USDT', 'ETH/USDT', 'LTC/USDT',
                                'TKN/USDT', 'ADA/USDT', 'ETC/USDT', ]
        # assert log_has_re(r'Removing pair .* since .* is below .*', caplog)

        # Move to "outside" of lookback window, so original sorting is restored.
        t.move_to("2021-09-01 07:00:00 +00:00")
        pm.refresh_pairlist()
        assert pm.whitelist == ['ETH/USDT', 'LTC/USDT', 'XRP/USDT',
                                'NEO/USDT', 'TKN/USDT', 'ADA/USDT', 'ETC/USDT']


def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None:
    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}]

    mocker.patch.multiple(EXMS,
                          get_tickers=tickers,
                          exchange_has=MagicMock(return_value=False),
                          )

    with pytest.raises(OperationalException,
                       match=r'Exchange does not support dynamic whitelist.*'):
        get_patched_freqtradebot(mocker, default_conf)


def test_pair_whitelist_not_supported_Spread(mocker, default_conf, tickers) -> None:
    default_conf['pairlists'] = [{'method': 'StaticPairList'}, {'method': 'SpreadFilter'}]

    mocker.patch.multiple(EXMS,
                          get_tickers=tickers,
                          exchange_has=MagicMock(return_value=False),
                          )

    with pytest.raises(OperationalException,
                       match=r'Exchange does not support fetchTickers, .*'):
        get_patched_freqtradebot(mocker, default_conf)

    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))
    mocker.patch(f'{EXMS}.get_option', MagicMock(return_value=False))
    with pytest.raises(OperationalException,
                       match=r'.*requires exchange to have bid/ask data'):
        get_patched_freqtradebot(mocker, default_conf)


@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
    whitelist_conf['pairlists'][0]['method'] = pairlist
    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True)
                          )
    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)

    assert freqtrade.pairlists.name_list == [pairlist]
    assert pairlist in str(freqtrade.pairlists.short_desc())
    assert isinstance(freqtrade.pairlists.whitelist, list)
    assert isinstance(freqtrade.pairlists.blacklist, list)


@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
@pytest.mark.parametrize("whitelist,log_message", [
    (['ETH/BTC', 'TKN/BTC'], ""),
    # TRX/ETH not in markets
    (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"),
    # wrong stake
    (['ETH/BTC', 'TKN/BTC', 'ETH/USDT'], "is not compatible with your stake currency"),
    # BCH/BTC not available
    (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"),
    # BTT/BTC is inactive
    (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active"),
    # XLTCUSDT is not a valid pair
    (['ETH/BTC', 'TKN/BTC', 'XLTCUSDT'], "is not tradable with Freqtrade"),
])
def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog,
                                       log_message, tickers):
    whitelist_conf['pairlists'][0]['method'] = pairlist
    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )
    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    caplog.clear()

    # Assign starting whitelist
    pairlist_handler = freqtrade.pairlists._pairlist_handlers[0]
    new_whitelist = pairlist_handler._whitelist_for_active_markets(whitelist)

    assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC'])
    assert log_message in caplog.text


@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, pairlist, tickers):
    whitelist_conf['pairlists'][0]['method'] = pairlist

    mocker.patch(f'{EXMS}.exchange_has', return_value=True)

    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=None),
                          get_tickers=tickers
                          )
    # Assign starting whitelist
    pairlist_handler = freqtrade.pairlists._pairlist_handlers[0]
    with pytest.raises(OperationalException, match=r'Markets not loaded.*'):
        pairlist_handler._whitelist_for_active_markets(['ETH/BTC'])


def test_volumepairlist_invalid_sortvalue(mocker, whitelist_conf):
    whitelist_conf['pairlists'][0].update({"sort_key": "asdf"})

    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))
    with pytest.raises(OperationalException,
                       match=r"key asdf not in .*"):
        get_patched_freqtradebot(mocker, whitelist_conf)


def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):

    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )
    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
    assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 0
    assert tickers.call_count == 0
    freqtrade.pairlists.refresh_pairlist()
    assert tickers.call_count == 1

    assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 1
    freqtrade.pairlists.refresh_pairlist()
    assert tickers.call_count == 1


def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers):
    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                 {'method': 'AgeFilter', 'min_days_listed': -1}]

    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )

    with pytest.raises(OperationalException,
                       match=r'AgeFilter requires min_days_listed to be >= 1'):
        get_patched_freqtradebot(mocker, default_conf)


def test_agefilter_max_days_lower_than_min_days(mocker, default_conf, markets, tickers):
    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                 {'method': 'AgeFilter', 'min_days_listed': 3,
                                 "max_days_listed": 2}]

    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )

    with pytest.raises(OperationalException,
                       match=r'AgeFilter max_days_listed <= min_days_listed not permitted'):
        get_patched_freqtradebot(mocker, default_conf)


def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers):
    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                 {'method': 'AgeFilter', 'min_days_listed': 99999}]

    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )

    with pytest.raises(OperationalException,
                       match=r'AgeFilter requires min_days_listed to not exceed '
                             r'exchange max request size \([0-9]+\)'):
        get_patched_freqtradebot(mocker, default_conf)


def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
        ohlcv_data = {
            ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history,
        }
        mocker.patch.multiple(
            EXMS,
            markets=PropertyMock(return_value=markets),
            exchange_has=MagicMock(return_value=True),
            get_tickers=tickers,
            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
        )

        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
        freqtrade.pairlists.refresh_pairlist()
        assert len(freqtrade.pairlists.whitelist) == 3
        assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0

        freqtrade.pairlists.refresh_pairlist()
        assert len(freqtrade.pairlists.whitelist) == 3
        # Call to XRP/BTC cached
        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2

        ohlcv_data = {
            ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history.iloc[[0]],
        }
        mocker.patch(f'{EXMS}.refresh_latest_ohlcv', return_value=ohlcv_data)
        freqtrade.pairlists.refresh_pairlist()
        assert len(freqtrade.pairlists.whitelist) == 3
        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1

        # Move to next day
        t.move_to("2021-09-02 01:00:00 +00:00")
        mocker.patch(f'{EXMS}.refresh_latest_ohlcv', return_value=ohlcv_data)
        freqtrade.pairlists.refresh_pairlist()
        assert len(freqtrade.pairlists.whitelist) == 3
        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1

        # Move another day with fresh mocks (now the pair is old enough)
        t.move_to("2021-09-03 01:00:00 +00:00")
        # Called once for XRP/BTC
        ohlcv_data = {
            ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history,
            ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history,
        }
        mocker.patch(f'{EXMS}.refresh_latest_ohlcv', return_value=ohlcv_data)
        freqtrade.pairlists.refresh_pairlist()
        assert len(freqtrade.pairlists.whitelist) == 4
        # Called once (only for XRP/BTC)
        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1


def test_OffsetFilter_error(mocker, whitelist_conf) -> None:
    whitelist_conf['pairlists'] = (
        [{"method": "StaticPairList"}, {"method": "OffsetFilter", "offset": -1}]
    )

    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))

    with pytest.raises(OperationalException,
                       match=r'OffsetFilter requires offset to be >= 0'):
        PairListManager(MagicMock, whitelist_conf)


def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                 {'method': 'RangeStabilityFilter', 'lookback_days': 99999}]

    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )

    with pytest.raises(OperationalException,
                       match=r'RangeStabilityFilter requires lookback_days to not exceed '
                             r'exchange max request size \([0-9]+\)'):
        get_patched_freqtradebot(mocker, default_conf)

    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                 {'method': 'RangeStabilityFilter', 'lookback_days': 0}]

    with pytest.raises(OperationalException,
                       match='RangeStabilityFilter requires lookback_days to be >= 1'):
        get_patched_freqtradebot(mocker, default_conf)


@pytest.mark.parametrize('min_rate_of_change,max_rate_of_change,expected_length', [
    (0.01, 0.99, 5),
    (0.05, 0.0, 0),  # Setting min rate_of_change to 5% removes all pairs from the whitelist.
])
def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history,
                                      min_rate_of_change, max_rate_of_change, expected_length):
    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                 {'method': 'RangeStabilityFilter', 'lookback_days': 2,
                                  'min_rate_of_change': min_rate_of_change,
                                  "max_rate_of_change": max_rate_of_change}]

    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )
    ohlcv_data = {
        ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('HOT/BTC', '1d', CandleType.SPOT): ohlcv_history,
        ('BLK/BTC', '1d', CandleType.SPOT): ohlcv_history,
    }
    mocker.patch.multiple(
        EXMS,
        refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
    )

    freqtrade = get_patched_freqtradebot(mocker, default_conf)
    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
    freqtrade.pairlists.refresh_pairlist()
    assert len(freqtrade.pairlists.whitelist) == expected_length
    assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0

    previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
    freqtrade.pairlists.refresh_pairlist()
    assert len(freqtrade.pairlists.whitelist) == expected_length
    # Should not have increased since first call.
    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count


def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplog):
    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                 {'method': 'SpreadFilter', 'max_spread_ratio': 0.1}]

    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          get_tickers=tickers
                          )

    ftbot = get_patched_freqtradebot(mocker, default_conf)
    ftbot.pairlists.refresh_pairlist()

    assert len(ftbot.pairlists.whitelist) == 5

    tickers.return_value['ETH/BTC']['ask'] = 0.0
    del tickers.return_value['TKN/BTC']
    del tickers.return_value['LTC/BTC']
    mocker.patch.multiple(EXMS, get_tickers=tickers)

    ftbot.pairlists.refresh_pairlist()
    assert log_has_re(r'Removed .* invalid ticker data.*', caplog)

    assert len(ftbot.pairlists.whitelist) == 2


@pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [
    ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010,
      "max_price": 1.0},
     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below "
     "0.1% or below 0.00000010 or above 1.00000000.'}]",
     None
     ),
    ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010},
     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.1% or below 0.00000010.'}]",
     None
     ),
    ({"method": "PriceFilter", "low_price_ratio": 0.001, "max_price": 1.00010000},
     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.1% or above 1.00010000.'}]",
     None
     ),
    ({"method": "PriceFilter", "min_price": 0.00002000},
     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.00002000.'}]",
     None
     ),
    ({"method": "PriceFilter", "max_value": 0.00002000},
     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced Value above 0.00002000.'}]",
     None
     ),
    ({"method": "PriceFilter"},
     "[{'PriceFilter': 'PriceFilter - No price filters configured.'}]",
     None
     ),
    ({"method": "PriceFilter", "low_price_ratio": -0.001},
     None,
     "PriceFilter requires low_price_ratio to be >= 0"
     ),  # OperationalException expected
    ({"method": "PriceFilter", "min_price": -0.00000010},
     None,
     "PriceFilter requires min_price to be >= 0"
     ),  # OperationalException expected
    ({"method": "PriceFilter", "max_price": -1.00010000},
     None,
     "PriceFilter requires max_price to be >= 0"
     ),  # OperationalException expected
    ({"method": "PriceFilter", "max_value": -1.00010000},
     None,
     "PriceFilter requires max_value to be >= 0"
     ),  # OperationalException expected
    ({"method": "RangeStabilityFilter", "lookback_days": 10,
      "min_rate_of_change": 0.01},
     "[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below "
     "0.01 over the last days.'}]",
        None
     ),
    ({"method": "RangeStabilityFilter", "lookback_days": 10,
     "min_rate_of_change": 0.01, "max_rate_of_change": 0.99},
     "[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below "
     "0.01 and above 0.99 over the last days.'}]",
        None
     ),
    ({"method": "OffsetFilter", "offset": 5, "number_assets": 10},
     "[{'OffsetFilter': 'OffsetFilter - Taking 10 Pairs, starting from 5.'}]",
     None
     ),
    ({"method": "ProducerPairList"},
     "[{'ProducerPairList': 'ProducerPairList - default'}]",
     None
     ),
])
def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
                          desc_expected, exception_expected):
    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True)
                          )
    whitelist_conf['pairlists'] = [pairlistconfig]

    if desc_expected is not None:
        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
        short_desc = str(freqtrade.pairlists.short_desc())
        assert short_desc == desc_expected
    else:  # OperationalException expected
        with pytest.raises(OperationalException,
                           match=exception_expected):
            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)


def test_pairlistmanager_no_pairlist(mocker, whitelist_conf):
    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))

    whitelist_conf['pairlists'] = []

    with pytest.raises(OperationalException,
                       match=r"No Pairlist Handlers defined"):
        get_patched_freqtradebot(mocker, whitelist_conf)


@pytest.mark.parametrize("pairlists,pair_allowlist,overall_performance,allowlist_result", [
    # No trades yet
    ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}],
     ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], [], ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
    # Happy path: Descending order, all values filled
    ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}],
     ['ETH/BTC', 'TKN/BTC'],
     [{'pair': 'TKN/BTC', 'profit_ratio': 0.05, 'count': 3},
      {'pair': 'ETH/BTC', 'profit_ratio': 0.04, 'count': 2}],
     ['TKN/BTC', 'ETH/BTC']),
    # Performance data outside allow list ignored
    ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}],
     ['ETH/BTC', 'TKN/BTC'],
     [{'pair': 'OTHER/BTC', 'profit_ratio': 0.05, 'count': 3},
      {'pair': 'ETH/BTC', 'profit_ratio': 0.04, 'count': 2}],
     ['ETH/BTC', 'TKN/BTC']),
    # Partial performance data missing and sorted between positive and negative profit
    ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}],
     ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'],
     [{'pair': 'ETH/BTC', 'profit_ratio': -0.05, 'count': 100},
      {'pair': 'TKN/BTC', 'profit_ratio': 0.04, 'count': 2}],
     ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']),
    # Tie in performance data broken by count (ascending)
    ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}],
     ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'],
     [{'pair': 'LTC/BTC', 'profit_ratio': -0.0501, 'count': 101},
      {'pair': 'TKN/BTC', 'profit_ratio': -0.0501, 'count': 2},
      {'pair': 'ETH/BTC', 'profit_ratio': -0.0501, 'count': 100}],
     ['TKN/BTC', 'ETH/BTC', 'LTC/BTC']),
    # Tie in performance and count, broken by prior sorting sort
    ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}],
     ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'],
     [{'pair': 'LTC/BTC', 'profit_ratio': -0.0501, 'count': 1},
      {'pair': 'TKN/BTC', 'profit_ratio': -0.0501, 'count': 1},
      {'pair': 'ETH/BTC', 'profit_ratio': -0.0501, 'count': 1}],
     ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
])
def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, overall_performance,
                            allowlist_result, tickers, markets, ohlcv_history_list):
    allowlist_conf = whitelist_conf
    allowlist_conf['pairlists'] = pairlists
    allowlist_conf['exchange']['pair_whitelist'] = pair_allowlist

    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))

    freqtrade = get_patched_freqtradebot(mocker, allowlist_conf)
    mocker.patch.multiple(EXMS,
                          get_tickers=tickers,
                          markets=PropertyMock(return_value=markets)
                          )
    mocker.patch.multiple(EXMS,
                          get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
                          )
    mocker.patch.multiple('freqtrade.persistence.Trade',
                          get_overall_performance=MagicMock(return_value=overall_performance),
                          )
    freqtrade.pairlists.refresh_pairlist()
    allowlist = freqtrade.pairlists.whitelist
    assert allowlist == allowlist_result


@pytest.mark.parametrize('wildcardlist,pairs,expected', [
    (['BTC/USDT'],
     ['BTC/USDT'],
     ['BTC/USDT']),
    (['BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETH/USDT']),
    (['BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT'], ['BTC/USDT']),  # Test one too many
    (['.*/USDT'],
     ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']),  # Wildcard simple
    (['.*C/USDT'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']),  # Wildcard exclude one
    (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
     ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']),  # Wildcard exclude one
    (['BTC/.*', 'ETH/.*'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'],
     ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']),  # Wildcard exclude one
    (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
     None),
    (['BTC/USD'],
     ['BTC/USD', 'BTC/USDT'],
     ['BTC/USD']),
])
def test_expand_pairlist(wildcardlist, pairs, expected):
    if expected is None:
        with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'):
            expand_pairlist(wildcardlist, pairs)
    else:
        assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected)
        conf = {
            'pairs': wildcardlist,
            'freqai': {
                "enabled": True,
                "feature_parameters": {
                    "include_corr_pairlist": [
                        "BTC/USDT:USDT",
                        "XRP/BUSD",
                    ]
                }
            }
        }
        assert sorted(dynamic_expand_pairlist(conf, pairs)) == sorted(expected + [
            "BTC/USDT:USDT",
            "XRP/BUSD",
        ])


@pytest.mark.parametrize('wildcardlist,pairs,expected', [
    (['BTC/USDT'],
     ['BTC/USDT'],
     ['BTC/USDT']),
    (['BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETH/USDT']),
    (['BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT'], ['BTC/USDT', 'ETH/USDT']),  # Test one too many
    (['.*/USDT'],
     ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']),  # Wildcard simple
    (['.*C/USDT'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']),  # Wildcard exclude one
    (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
     ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']),  # Wildcard exclude one
    (['BTC/.*', 'ETH/.*'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'],
     ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']),  # Wildcard exclude one
    (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
     None),
    (['HELLO/WORLD'], [], ['HELLO/WORLD']),  # Invalid pair kept
    (['BTC/USD'],
     ['BTC/USD', 'BTC/USDT'],
     ['BTC/USD']),

])
def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected):
    if expected is None:
        with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'):
            expand_pairlist(wildcardlist, pairs, keep_invalid=True)
    else:
        assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected)


def test_ProducerPairlist_no_emc(mocker, whitelist_conf):
    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))

    whitelist_conf['pairlists'] = [
        {
            "method": "ProducerPairList",
            "number_assets": 10,
            "producer_name": "hello_world",
        }
    ]
    del whitelist_conf['external_message_consumer']

    with pytest.raises(OperationalException,
                       match=r"ProducerPairList requires external_message_consumer to be enabled."):
        get_patched_freqtradebot(mocker, whitelist_conf)


def test_ProducerPairlist(mocker, whitelist_conf, markets):
    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))
    mocker.patch.multiple(EXMS,
                          markets=PropertyMock(return_value=markets),
                          exchange_has=MagicMock(return_value=True),
                          )
    whitelist_conf['pairlists'] = [
        {
            "method": "ProducerPairList",
            "number_assets": 2,
            "producer_name": "hello_world",
        }
    ]
    whitelist_conf.update({
        "external_message_consumer": {
            "enabled": True,
            "producers": [
                {
                    "name": "hello_world",
                    "host": "null",
                    "port": 9891,
                    "ws_token": "dummy",
                }
            ]
        }
    })

    exchange = get_patched_exchange(mocker, whitelist_conf)
    dp = DataProvider(whitelist_conf, exchange, None)
    pairs = ['ETH/BTC', 'LTC/BTC', 'XRP/BTC']
    # different producer
    dp._set_producer_pairs(pairs + ['MEEP/USDT'], 'default')
    pm = PairListManager(exchange, whitelist_conf, dp)
    pm.refresh_pairlist()
    assert pm.whitelist == []
    # proper producer
    dp._set_producer_pairs(pairs, 'hello_world')
    pm.refresh_pairlist()

    # Pairlist reduced to 2
    assert pm.whitelist == pairs[:2]
    assert len(pm.whitelist) == 2
    whitelist_conf['exchange']['pair_whitelist'] = ['TKN/BTC']

    whitelist_conf['pairlists'] = [
        {"method": "StaticPairList"},
        {
            "method": "ProducerPairList",
            "producer_name": "hello_world",
        }
    ]
    pm = PairListManager(exchange, whitelist_conf, dp)
    pm.refresh_pairlist()
    assert len(pm.whitelist) == 4
    assert pm.whitelist == ['TKN/BTC'] + pairs