From 4bfab5e222a774416e43f9db58e5d8742f4f99ef Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 18 May 2020 02:36:40 +0300 Subject: [PATCH 1/6] Add ShuffleFilter --- freqtrade/constants.py | 2 +- freqtrade/pairlist/ShuffleFilter.py | 50 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 freqtrade/pairlist/ShuffleFilter.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e56586bbc..5d3b13eee 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -19,7 +19,7 @@ ORDERBOOK_SIDES = ['ask', 'bid'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'PrecisionFilter', 'PriceFilter', 'SpreadFilter'] + 'PrecisionFilter', 'PriceFilter', 'ShuffleFilter', 'SpreadFilter'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons diff --git a/freqtrade/pairlist/ShuffleFilter.py b/freqtrade/pairlist/ShuffleFilter.py new file mode 100644 index 000000000..a10a1af8b --- /dev/null +++ b/freqtrade/pairlist/ShuffleFilter.py @@ -0,0 +1,50 @@ +""" +Shuffle pair list filter +""" +import logging +import random +from typing import Dict, List + +from freqtrade.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class ShuffleFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._seed = pairlistconfig.get('seed') + self._random = random.Random(self._seed) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Shuffling pairs" + + (f", seed = {self._seed}." if self._seed is not None else ".")) + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + # Shuffle is done inplace + self._random.shuffle(pairlist) + + return pairlist From 287e8bafce02e1881fc9318a082bd7b7f781e0ec Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 18 May 2020 02:37:03 +0300 Subject: [PATCH 2/6] Add/adjust tests --- tests/pairlist/test_pairlist.py | 66 +++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 61f6b4bd5..929d87779 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -157,16 +157,28 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ + # VolumePairList only ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), # Different sorting depending on quote or bid volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), - # No pair for ETH ... + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), + # No pair for ETH, VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "ETH", []), + # No pair for ETH, StaticPairList + ([{"method": "StaticPairList"}], + "ETH", []), + # No pair for ETH, all handlers + ([{"method": "StaticPairList"}, + {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "PrecisionFilter"}, + {"method": "PriceFilter", "low_price_ratio": 0.03}, + {"method": "SpreadFilter", "max_spread": 0.005}, + {"method": "ShuffleFilter"}], + "ETH", []), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), @@ -176,31 +188,43 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "USDT", ['ETH/USDT', 'NANO/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT']), # Hot is removed by precision_filter, Fuel by low_price_filter. ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, - {"method": "PriceFilter", "low_price_ratio": 0.02} - ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + {"method": "PriceFilter", "low_price_ratio": 0.02}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # HOT and XRP are removed because below 1250 quoteVolume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "min_value": 1250}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), - # StaticPairlist Only - ([{"method": "StaticPairList"}, - ], "BTC", ['ETH/BTC', 'TKN/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + # StaticPairlist only + ([{"method": "StaticPairList"}], + "BTC", ['ETH/BTC', 'TKN/BTC']), # Static Pairlist before VolumePairList - sorting changes ([{"method": "StaticPairList"}, - {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, - ], "BTC", ['TKN/BTC', 'ETH/BTC']), + {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], + "BTC", ['TKN/BTC', 'ETH/BTC']), # SpreadFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "SpreadFilter", "max_spread": 0.005} - ], "USDT", ['ETH/USDT']), + {"method": "SpreadFilter", "max_spread": 0.005}], + "USDT", ['ETH/USDT']), + # ShuffleFilter + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "ShuffleFilter", "seed": 77}], + "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']), + # ShuffleFilter, other seed + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "ShuffleFilter", "seed": 42}], + "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']), + # ShuffleFilter, no seed + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "ShuffleFilter"}], + "USDT", 3), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, pairlists, base_currency, whitelist_result, @@ -219,12 +243,16 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist - assert whitelist == whitelist_result + if type(whitelist_result) is list: + assert whitelist == whitelist_result + else: + len(whitelist) == whitelist_result + for pairlist in pairlists: - if pairlist['method'] == 'PrecisionFilter': + if pairlist['method'] == 'PrecisionFilter' and whitelist_result: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) - if pairlist['method'] == 'PriceFilter': + if pairlist['method'] == 'PriceFilter' and whitelist_result: assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] is empty.*", caplog)) From 51c0639e6dd1d55ae29e25973ca0175d69c3ede8 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Mon, 18 May 2020 11:54:52 +0300 Subject: [PATCH 3/6] Update tests/pairlist/test_pairlist.py Co-authored-by: Matthias --- tests/pairlist/test_pairlist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 929d87779..3a8f94974 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -243,6 +243,9 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist + assert isinstance(whitelist, list) + + # Verify length of pairlist matches (used for ShuffleFilter without seed) if type(whitelist_result) is list: assert whitelist == whitelist_result else: From f54dc7affd54fbb1b7933ecdb776e53c2e033a2d Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Mon, 18 May 2020 13:18:05 +0300 Subject: [PATCH 4/6] Make flake happy --- tests/pairlist/test_pairlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 3a8f94974..e013122e4 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -244,7 +244,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t whitelist = freqtrade.pairlists.whitelist assert isinstance(whitelist, list) - + # Verify length of pairlist matches (used for ShuffleFilter without seed) if type(whitelist_result) is list: assert whitelist == whitelist_result From 30d1a8589553c42dafffde391457d5466c018ef3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 18 May 2020 23:46:23 +0300 Subject: [PATCH 5/6] Adjust docs --- docs/configuration.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 87005ac74..93e53de6f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -555,7 +555,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). -Additionaly, [`PrecisionFilter`](#precision-filter), [`PriceFilter`](#price-pair-filter) and [`SpreadFilter`](#spread-pair-filter) act as Pairlist Filters, removing certain pairs. +Additionaly, [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler. @@ -565,9 +565,10 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) -* [`PrecisionFilter`](#precision-filter) -* [`PriceFilter`](#price-pair-filter) -* [`SpreadFilter`](#spread-filter) +* [`PrecisionFilter`](#precisionfilter) +* [`PriceFilter`](#pricefilter) +* [`ShuffleFilter`](#shufflefilter) +* [`SpreadFilter`](#spreadfilter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility subcommand to test your configuration quickly. @@ -624,17 +625,24 @@ Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0. These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses. Here is what the PriceFilters takes over. +#### ShuffleFilter + +Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. + +!!! Tip + You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. + #### SpreadFilter -Removes pairs that have a difference between asks and bids above the specified ratio (default `0.005`). +Removes pairs that have a difference between asks and bids above the specified ratio, `max_spread_ratio` (defaults to `0.005`). Example: -If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027 the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` +If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out. ### Full example of Pairlist Handlers -The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter), filtering all assets where 1 priceunit is > 1%. +The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 priceunit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. ```json "exchange": { @@ -648,7 +656,9 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "sort_key": "quoteVolume", }, {"method": "PrecisionFilter"}, - {"method": "PriceFilter", "low_price_ratio": 0.01} + {"method": "PriceFilter", "low_price_ratio": 0.01}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005}, + {"method": "ShuffleFilter", "seed": 42} ], ``` From d8352bd6324561ea75b6064ad08b1c3751dccfa4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 18 May 2020 23:48:06 +0300 Subject: [PATCH 6/6] Fix tests for SpreadFilter --- tests/pairlist/test_pairlist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index e013122e4..c4893d7e4 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -176,7 +176,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.03}, - {"method": "SpreadFilter", "max_spread": 0.005}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005}, {"method": "ShuffleFilter"}], "ETH", []), # Precisionfilter and quote volume @@ -211,7 +211,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "BTC", ['TKN/BTC', 'ETH/BTC']), # SpreadFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "SpreadFilter", "max_spread": 0.005}], + {"method": "SpreadFilter", "max_spread_ratio": 0.005}], "USDT", ['ETH/USDT']), # ShuffleFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},