diff --git a/config_full.json.example b/config_full.json.example index 45c5c695c..5ee2a1faf 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -67,7 +67,13 @@ {"method": "AgeFilter", "min_days_listed": 10}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010}, - {"method": "SpreadFilter", "max_spread_ratio": 0.005} + {"method": "SpreadFilter", "max_spread_ratio": 0.005}, + { + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, + "refresh_period": 1440 + } ], "exchange": { "name": "bittrex", diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index e6a9fc1a8..149e784bd 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -19,6 +19,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac * [`PriceFilter`](#pricefilter) * [`ShuffleFilter`](#shufflefilter) * [`SpreadFilter`](#spreadfilter) +* [`RangeStabilityFilter`](#rangestabilityfilter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly. @@ -118,6 +119,27 @@ Example: If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out. +#### RangeStabilityFilter + +Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. + +In the below example: +If the trading range over the last 10 days is <1%, remove the pair from the whitelist. + +```json +"pairlists": [ + { + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, + "refresh_period": 1440 + } +] +``` + +!!! Tip + This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit. + ### Full example of Pairlist Handlers The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. @@ -137,6 +159,12 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, + { + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, + "refresh_period": 1440 + }, {"method": "ShuffleFilter", "seed": 42} ], ``` diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3271dda39..2022556d2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -25,7 +25,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PrecisionFilter', 'PriceFilter', - 'ShuffleFilter', 'SpreadFilter'] + 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2bbdb0d59..18f4fbff5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -679,12 +679,25 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :returns List with candle (OHLCV) data + :return: List with candle (OHLCV) data """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms)) + def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, + since_ms: int) -> DataFrame: + """ + Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe + :param pair: Pair to download + :param timeframe: Timeframe to get data for + :param since_ms: Timestamp in milliseconds to get history from + :return: OHLCV DataFrame + """ + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) + return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: diff --git a/freqtrade/pairlist/AgeFilter.py b/freqtrade/pairlist/AgeFilter.py index 20635a9ed..e2a13c20a 100644 --- a/freqtrade/pairlist/AgeFilter.py +++ b/freqtrade/pairlist/AgeFilter.py @@ -49,7 +49,7 @@ class AgeFilter(IPairList): return (f"{self.name} - Filtering pairs with age less than " f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") - def _validate_pair(self, ticker: dict) -> bool: + def _validate_pair(self, ticker: Dict) -> bool: """ Validate age for the ticker :param ticker: ticker dict as returned from ccxt.load_markets() diff --git a/freqtrade/pairlist/rangestabilityfilter.py b/freqtrade/pairlist/rangestabilityfilter.py new file mode 100644 index 000000000..b460ff477 --- /dev/null +++ b/freqtrade/pairlist/rangestabilityfilter.py @@ -0,0 +1,89 @@ +""" +Rate of change pairlist filter +""" +import logging +from typing import Any, Dict + +import arrow +from cachetools.ttl import TTLCache + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import plural +from freqtrade.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class RangeStabilityFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('lookback_days', 10) + self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) + self._refresh_period = pairlistconfig.get('refresh_period', 1440) + + self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period) + + if self._days < 1: + raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1") + if self._days > exchange.ohlcv_candle_limit: + raise OperationalException("RangeStabilityFilter requires lookback_days to not " + "exceed exchange max request size " + f"({exchange.ohlcv_candle_limit})") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with rate of change below " + f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.") + + def _validate_pair(self, ticker: Dict) -> bool: + """ + Validate trading range + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, False if it should be removed + """ + pair = ticker['symbol'] + # Check symbol in cache + if pair in self._pair_cache: + return self._pair_cache[pair] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days) + .float_timestamp) * 1000 + + daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair, + timeframe='1d', + since_ms=since_ms) + result = False + if daily_candles is not None and not daily_candles.empty: + highest_high = daily_candles['high'].max() + lowest_low = daily_candles['low'].min() + pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 + if pct_change >= self._min_rate_of_change: + result = True + else: + self.log_on_refresh(logger.info, + f"Removed {pair} from whitelist, " + f"because rate of change over {plural(self._days, 'day')} is " + f"{pct_change:.3f}, which is below the " + f"threshold of {self._min_rate_of_change}.") + result = False + self._pair_cache[pair] = result + + return result diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e4452a83c..42681b367 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1307,6 +1307,57 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): assert log_has_re(r"Async code raised an exception: .*", caplog) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + ohlcv = [ + [ + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ], + [ + arrow.utcnow().shift(minutes=5).int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ], + [ + arrow.utcnow().shift(minutes=10).int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + pair = 'ETH/BTC' + + async def mock_candle_hist(pair, timeframe, since_ms): + return pair, timeframe, ohlcv + + exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) + # one_call calculation * 1.8 should do 2 calls + + since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int(( + arrow.utcnow().int_timestamp - since) * 1000)) + + assert exchange._async_get_candle_history.call_count == 2 + # Returns twice the above OHLCV data + assert len(ret) == 2 + assert isinstance(ret, DataFrame) + assert 'date' in ret.columns + assert 'open' in ret.columns + assert 'close' in ret.columns + assert 'high' in ret.columns + + def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: ohlcv = [ [ diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 1f05bef1e..d696e6d02 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -58,7 +58,7 @@ def whitelist_conf_2(default_conf): @pytest.fixture(scope="function") -def whitelist_conf_3(default_conf): +def whitelist_conf_agefilter(default_conf): default_conf['stake_currency'] = 'BTC' default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', @@ -340,6 +340,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], "USDT", ['ETH/USDT', 'NANO/USDT']), + ([{"method": "StaticPairList"}, + {"method": "RangeStabilityFilter", "lookback_days": 10, + "min_rate_of_change": 0.01, "refresh_period": 1440}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history_list, pairlists, base_currency, @@ -528,7 +532,7 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf -def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers, caplog): +def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'AgeFilter', 'min_days_listed': -1}] @@ -543,7 +547,7 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) -def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers, caplog): +def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'AgeFilter', 'min_days_listed': 99999}] @@ -559,7 +563,7 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) -def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_history_list): +def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history_list): mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), @@ -571,7 +575,7 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), ) - freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_3) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) assert freqtrade.exchange.get_historic_ohlcv.call_count == 0 freqtrade.pairlists.refresh_pairlist() assert freqtrade.exchange.get_historic_ohlcv.call_count > 0 @@ -582,6 +586,62 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count +def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'lookback_days': 99999}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'RangeStabilityFilter requires lookback_days to not exceed ' + r'exchange max request size \([0-9]+\)'): + get_patched_freqtradebot(mocker, default_conf) + + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'lookback_days': 0}] + + with pytest.raises(OperationalException, + match='RangeStabilityFilter requires lookback_days to be >= 1'): + get_patched_freqtradebot(mocker, default_conf) + + +@pytest.mark.parametrize('min_rate_of_change,expected_length', [ + (0.01, 5), + (0.05, 0), # Setting rate_of_change to 5% removes all pairs from the whitelist. +]) +def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list, + min_rate_of_change, expected_length): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'lookback_days': 2, + 'min_rate_of_change': min_rate_of_change}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.exchange.get_historic_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == expected_length + assert freqtrade.exchange.get_historic_ohlcv.call_count > 0 + + previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == expected_length + # Should not have increased since first call. + assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count + + @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [ ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010, "max_price": 1.0}, @@ -617,6 +677,11 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his None, "PriceFilter requires max_price to be >= 0" ), # OperationalException expected + ({"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01}, + "[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below " + "0.01 over the last days.'}]", + None + ), ]) def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, desc_expected, exception_expected):