diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 5bb02470d..844f1d70a 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -15,6 +15,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) * [`AgeFilter`](#agefilter) +* [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) * [`PriceFilter`](#pricefilter) * [`ShuffleFilter`](#shufflefilter) @@ -74,6 +75,15 @@ be caught out buying before the pair has finished dropping in price. This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days. +#### PerformanceFilter + +Sorts pairs by past trade performance, as follows: +1. Positive performance. +2. No closed trades yet. +3. Negative performance. + +Trade count is used as a tie breaker. + #### PrecisionFilter Filters low-value coins which would not allow setting stoplosses. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3e523a49e..601e525c1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,8 +24,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'AgeFilter', 'PrecisionFilter', 'PriceFilter', - 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] + 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', + 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', + 'SpreadFilter'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/pairlist/PerformanceFilter.py b/freqtrade/pairlist/PerformanceFilter.py new file mode 100644 index 000000000..92a97099e --- /dev/null +++ b/freqtrade/pairlist/PerformanceFilter.py @@ -0,0 +1,66 @@ +""" +Performance pair list filter +""" +import logging +from typing import Any, Dict, List + +import pandas as pd + +from freqtrade.pairlist.IPairList import IPairList +from freqtrade.persistence import Trade + + +logger = logging.getLogger(__name__) + + +class PerformanceFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short allowlist method description - used for startup-messages + """ + return f"{self.name} - Sorting pairs by performance." + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the allowlist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + # Get the trading performance for pairs from database + performance = pd.DataFrame(Trade.get_overall_performance()) + + # Skip performance-based sorting if no performance data is available + if len(performance) == 0: + return pairlist + + # Get pairlist from performance dataframe values + list_df = pd.DataFrame({'pair': pairlist}) + + # Set initial value for pairs with no trades to 0 + # Sort the list using: + # - primarily performance (high to low) + # - then count (low to high, so as to favor same performance with fewer trades) + # - then pair name alphametically + sorted_df = list_df.merge(performance, on='pair', how='left')\ + .fillna(0).sort_values(by=['count', 'pair'], ascending=True)\ + .sort_values(by=['profit'], ascending=False) + pairlist = sorted_df['pair'].tolist() + + return pairlist diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index d696e6d02..1d2f16b45 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -246,7 +246,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, - {"method": "ShuffleFilter"}], + {"method": "ShuffleFilter"}, {"method": "PerformanceFilter"}], "ETH", []), # AgeFilter and VolumePairList (require 2 days only, all should pass age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, @@ -326,6 +326,13 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # ShuffleFilter only ([{"method": "ShuffleFilter", "seed": 42}], "BTC", 'filter_at_the_beginning'), # OperationalException expected + # PerformanceFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "PerformanceFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), + # PerformanceFilter only + ([{"method": "PerformanceFilter"}], + "BTC", 'filter_at_the_beginning'), # OperationalException expected # SpreadFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], @@ -370,6 +377,11 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), ) + # Provide for PerformanceFilter's dependency + mocker.patch.multiple('freqtrade.persistence.Trade', + get_overall_performance=MagicMock(return_value=[]) + ) + # Set whitelist_result to None if pairlist is invalid and should produce exception if whitelist_result == 'filter_at_the_beginning': with pytest.raises(OperationalException, @@ -413,7 +425,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert not log_has(logmsg, caplog) -def test_PrecisionFilter_error(mocker, whitelist_conf, tickers) -> None: +def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}] del whitelist_conf['stoploss'] @@ -486,7 +498,7 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) -def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, markets, pairlist, tickers): +def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, pairlist, tickers): whitelist_conf['pairlists'][0]['method'] = pairlist mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True) @@ -502,7 +514,7 @@ def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, markets, pa pairlist_handler._whitelist_for_active_markets(['ETH/BTC']) -def test_volumepairlist_invalid_sortvalue(mocker, markets, whitelist_conf): +def test_volumepairlist_invalid_sortvalue(mocker, whitelist_conf): whitelist_conf['pairlists'][0].update({"sort_key": "asdf"}) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -701,7 +713,7 @@ def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) -def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): +def test_pairlistmanager_no_pairlist(mocker, whitelist_conf): mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) whitelist_conf['pairlists'] = [] @@ -709,3 +721,63 @@ def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): with pytest.raises(OperationalException, match=r"No Pairlist Handlers defined"): get_patched_freqtradebot(mocker, whitelist_conf) + + +@pytest.mark.parametrize("pairlists,pair_allowlist,overall_performance,allowlist_result", [ + # No trades yet + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], [], ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + # Happy path: Descending order, all values filled + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC'], + [{'pair': 'TKN/BTC', 'profit': 5, 'count': 3}, {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + ['TKN/BTC', 'ETH/BTC']), + # Performance data outside allow list ignored + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC'], + [{'pair': 'OTHER/BTC', 'profit': 5, 'count': 3}, + {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + ['ETH/BTC', 'TKN/BTC']), + # Partial performance data missing and sorted between positive and negative profit + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'ETH/BTC', 'profit': -5, 'count': 100}, + {'pair': 'TKN/BTC', 'profit': 4, 'count': 2}], + ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']), + # Tie in performance data broken by count (ascending) + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 101}, + {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 2}, + {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 100}], + ['TKN/BTC', 'ETH/BTC', 'LTC/BTC']), + # Tie in performance and count, broken by alphabetical sort + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 1}, + {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 1}, + {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 1}], + ['ETH/BTC', 'LTC/BTC', 'TKN/BTC']), +]) +def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, overall_performance, + allowlist_result, tickers, markets, ohlcv_history_list): + allowlist_conf = whitelist_conf + allowlist_conf['pairlists'] = pairlists + allowlist_conf['exchange']['pair_whitelist'] = pair_allowlist + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + freqtrade = get_patched_freqtradebot(mocker, allowlist_conf) + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=markets) + ) + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + ) + mocker.patch.multiple('freqtrade.persistence.Trade', + get_overall_performance=MagicMock(return_value=overall_performance), + ) + freqtrade.pairlists.refresh_pairlist() + allowlist = freqtrade.pairlists.whitelist + assert allowlist == allowlist_result