From 3d9f3eeb07bc67e08d7ce9b3ba1506a422dae774 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sat, 3 Jul 2021 23:58:04 +0700 Subject: [PATCH 1/8] feat(agefilter): add max_days_listed --- config_full.json.example | 2 +- docs/includes/pairlists.md | 4 +-- freqtrade/plugins/pairlist/AgeFilter.py | 17 +++++++++--- tests/plugins/test_pairlist.py | 37 +++++++++++++++++++++---- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index d404391a4..6df4a8253 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -67,7 +67,7 @@ "sort_key": "quoteVolume", "refresh_period": 1800 }, - {"method": "AgeFilter", "min_days_listed": 10}, + {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index f19c5a181..b07bde62f 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -87,7 +87,7 @@ When pairs are first listed on an exchange they can suffer huge price drops and in the first few days while the pair goes through its price-discovery period. Bots can often be caught out buying before the pair has finished dropping in price. -This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days. +This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`. #### PerformanceFilter @@ -212,7 +212,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "number_assets": 20, "sort_key": "quoteVolume" }, - {"method": "AgeFilter", "min_days_listed": 10}, + {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 8f623b062..56643cca8 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -27,6 +27,7 @@ class AgeFilter(IPairList): super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) + self._max_days_listed = pairlistconfig.get('max_days_listed', None) if self._min_days_listed < 1: raise OperationalException("AgeFilter requires min_days_listed to be >= 1") @@ -34,6 +35,8 @@ class AgeFilter(IPairList): raise OperationalException("AgeFilter requires min_days_listed to not exceed " "exchange max request size " f"({exchange.ohlcv_candle_limit('1d')})") + if self._max_days_listed and self._max_days_listed <= self._min_days_listed: + raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted") @property def needstickers(self) -> bool: @@ -49,7 +52,9 @@ class AgeFilter(IPairList): Short whitelist method description - used for startup-messages """ return (f"{self.name} - Filtering pairs with age less than " - f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") + f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}" + " or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}") def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ @@ -61,9 +66,10 @@ class AgeFilter(IPairList): if not needed_pairs: return pairlist + since_days = -(self._max_days_listed if self._max_days_listed else self._min_days_listed) - 1 since_ms = int(arrow.utcnow() .floor('day') - .shift(days=-self._min_days_listed - 1) + .shift(days=since_days) .float_timestamp) * 1000 candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: @@ -86,7 +92,8 @@ class AgeFilter(IPairList): return True if daily_candles is not None: - if len(daily_candles) >= self._min_days_listed: + if len(daily_candles) >= self._min_days_listed and \ + len(daily_candles) <= self._max_days_listed: # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 @@ -94,6 +101,8 @@ class AgeFilter(IPairList): else: self.log_once(f"Removed {pair} from whitelist, because age " f"{len(daily_candles)} is less than {self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')}", logger.info) + f"{plural(self._min_days_listed, 'day')} or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}", + logger.info) return False return False diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index ae8f6e958..03d8fc563 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -79,7 +79,8 @@ def whitelist_conf_agefilter(default_conf): }, { "method": "AgeFilter", - "min_days_listed": 2 + "min_days_listed": 2, + "max_days_listed": 100 } ] return default_conf @@ -302,7 +303,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # No pair for ETH, all handlers ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 2}, + {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": None}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, @@ -310,11 +311,15 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "ETH", []), # AgeFilter and VolumePairList (require 2 days only, all should pass age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 2}], + {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": 100}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), # AgeFilter and VolumePairList (require 10 days, all should fail age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 10}], + {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": None}], + "BTC", []), + # AgeFilter and VolumePairList (all pair listed > 2, all should fail age test) + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}], "BTC", []), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, @@ -480,9 +485,13 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t for pairlist in pairlists: if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \ - len(ohlcv_history) <= pairlist['min_days_listed']: + len(ohlcv_history) < pairlist['min_days_listed']: assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' - r'.* day.*', caplog) + r'.* day.* or more than .* day', caplog) + if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \ + len(ohlcv_history) > pairlist['max_days_listed']: + assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' + r'.* day.* or more than .* day', caplog) if pairlist['method'] == 'PrecisionFilter' and whitelist_result: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) @@ -650,6 +659,22 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) +def test_agefilter_max_days_lower_than_min_days(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'AgeFilter', 'min_days_listed': 3, + "max_days_listed": 2}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'AgeFilter max_days_listed <= min_days_listed not permitted'): + get_patched_freqtradebot(mocker, default_conf) + + def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'AgeFilter', 'min_days_listed': 99999}] From b72bbebccbe5bec7ad920122b1cd7bdfa1541246 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 01:46:51 +0700 Subject: [PATCH 2/8] fix flake8 --- freqtrade/plugins/pairlist/AgeFilter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 56643cca8..e792e0343 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -66,7 +66,9 @@ class AgeFilter(IPairList): if not needed_pairs: return pairlist - since_days = -(self._max_days_listed if self._max_days_listed else self._min_days_listed) - 1 + since_days = -( + self._max_days_listed if self._max_days_listedelse else self._min_days_listed + ) - 1 since_ms = int(arrow.utcnow() .floor('day') .shift(days=since_days) From f6511c3e3f75597cdc725e4ba9c4cb078291e680 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 02:20:53 +0700 Subject: [PATCH 3/8] fix typo and add blocker --- freqtrade/plugins/pairlist/AgeFilter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index e792e0343..63a9ecfeb 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -37,6 +37,10 @@ class AgeFilter(IPairList): f"({exchange.ohlcv_candle_limit('1d')})") if self._max_days_listed and self._max_days_listed <= self._min_days_listed: raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted") + if self._max_days_listed > exchange.ohlcv_candle_limit('1d'): + raise OperationalException("AgeFilter requires max_days_listed to not exceed " + "exchange max request size " + f"({exchange.ohlcv_candle_limit('1d')})") @property def needstickers(self) -> bool: @@ -67,7 +71,7 @@ class AgeFilter(IPairList): return pairlist since_days = -( - self._max_days_listed if self._max_days_listedelse else self._min_days_listed + self._max_days_listed if self._max_days_listed else self._min_days_listed ) - 1 since_ms = int(arrow.utcnow() .floor('day') From 2d5ced780116088aca71216c29683021754b22dd Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 21:59:59 +0700 Subject: [PATCH 4/8] fix testcase --- config_full.json.example | 2 +- docs/includes/pairlists.md | 4 ++-- freqtrade/plugins/pairlist/AgeFilter.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 6df4a8253..d404391a4 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -67,7 +67,7 @@ "sort_key": "quoteVolume", "refresh_period": 1800 }, - {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300}, + {"method": "AgeFilter", "min_days_listed": 10}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index b07bde62f..908eaa459 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -81,7 +81,7 @@ Filtering instances (not the first position in the list) will not apply any cach #### AgeFilter -Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`). +Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity). When pairs are first listed on an exchange they can suffer huge price drops and volatility in the first few days while the pair goes through its price-discovery period. Bots can often @@ -212,7 +212,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "number_assets": 20, "sort_key": "quoteVolume" }, - {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300}, + {"method": "AgeFilter", "min_days_listed": 10}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 63a9ecfeb..ef3953776 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -37,7 +37,7 @@ class AgeFilter(IPairList): f"({exchange.ohlcv_candle_limit('1d')})") if self._max_days_listed and self._max_days_listed <= self._min_days_listed: raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted") - if self._max_days_listed > exchange.ohlcv_candle_limit('1d'): + if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'): raise OperationalException("AgeFilter requires max_days_listed to not exceed " "exchange max request size " f"({exchange.ohlcv_candle_limit('1d')})") From 502c69dce3b2e9b8d95e87bc293948165b5e34fc Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Tue, 6 Jul 2021 19:36:42 +0700 Subject: [PATCH 5/8] change short desc --- freqtrade/plugins/pairlist/AgeFilter.py | 24 +++++++++++++++--------- tests/plugins/test_pairlist.py | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index ef3953776..c482f6217 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -55,10 +55,13 @@ class AgeFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return (f"{self.name} - Filtering pairs with age less than " - f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}" - " or more than " - f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}") + return ( + f"{self.name} - Filtering pairs with age less than " + f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}" + ) + ( + " or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" + ) if self._max_days_listed else '' def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ @@ -105,10 +108,13 @@ class AgeFilter(IPairList): self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 return True else: - self.log_once(f"Removed {pair} from whitelist, because age " - f"{len(daily_candles)} is less than {self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')} or more than " - f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}", - logger.info) + self.log_once(( + f"Removed {pair} from whitelist, because age " + f"{len(daily_candles)} is less than {self._min_days_listed} " + f"{plural(self._min_days_listed, 'day')}" + ) + ( + " or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" + ) if self.max_days_listed else '', logger.info) return False return False diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 03d8fc563..5724ca39c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -487,7 +487,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \ len(ohlcv_history) < pairlist['min_days_listed']: assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' - r'.* day.* or more than .* day', caplog) + r'.* day.*', caplog) if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \ len(ohlcv_history) > pairlist['max_days_listed']: assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' From 00a1931f40e1351191fb030cdd649671aee7380c Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Wed, 7 Jul 2021 21:24:44 +0700 Subject: [PATCH 6/8] fix test --- freqtrade/plugins/pairlist/AgeFilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 44c5d2e5e..810a87082 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -102,7 +102,7 @@ class AgeFilter(IPairList): if daily_candles is not None: if len(daily_candles) >= self._min_days_listed and \ - len(daily_candles) <= self._max_days_listed: + (True if not self._max_days_listed else len(daily_candles) <= self._max_days_listed): # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 @@ -112,9 +112,9 @@ class AgeFilter(IPairList): f"Removed {pair} from whitelist, because age " f"{len(daily_candles)} is less than {self._min_days_listed} " f"{plural(self._min_days_listed, 'day')}" - ) + ( + ) + (( " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" - ) if self.max_days_listed else '', logger.info) + ) if self._max_days_listed else ''), logger.info) return False return False From 8248d1acd19c1b229fecd894d480bbd320912fc3 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Wed, 7 Jul 2021 22:10:22 +0700 Subject: [PATCH 7/8] run flake8 --- freqtrade/plugins/pairlist/AgeFilter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 810a87082..133285884 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -102,7 +102,8 @@ class AgeFilter(IPairList): if daily_candles is not None: if len(daily_candles) >= self._min_days_listed and \ - (True if not self._max_days_listed else len(daily_candles) <= self._max_days_listed): + (True if not self._max_days_listed + else len(daily_candles) <= self._max_days_listed): # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 From 682f880630dbf31bd99cf604d94256c989f90df1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Jul 2021 20:05:56 +0200 Subject: [PATCH 8/8] Slightly simplify if statement, add additional test --- freqtrade/plugins/pairlist/AgeFilter.py | 7 ++++--- tests/plugins/test_pairlist.py | 10 +++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 133285884..23250e5c1 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -101,9 +101,10 @@ class AgeFilter(IPairList): return True if daily_candles is not None: - if len(daily_candles) >= self._min_days_listed and \ - (True if not self._max_days_listed - else len(daily_candles) <= self._max_days_listed): + if ( + len(daily_candles) >= self._min_days_listed + and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed) + ): # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5724ca39c..f8c5acba3 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -321,6 +321,14 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}], "BTC", []), + # AgeFilter and VolumePairList LTC/BTC has 6 candles - removes all + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 5}], + "BTC", []), + # AgeFilter and VolumePairList LTC/BTC has 6 candles - passes + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 10}], + "BTC", ["LTC/BTC"]), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], @@ -436,7 +444,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ohlcv_data = { ('ETH/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history.append(ohlcv_history), ('XRP/BTC', '1d'): ohlcv_history, ('HOT/BTC', '1d'): ohlcv_history_high_vola, }