Merge pull request #3975 from freqtrade/volatility_filter

RangeStabilityFilter filter - filter for pairs without much movement

config_full.json.example

@@ -67,7 +67,13 @@
         {"method": "AgeFilter", "min_days_listed": 10},
         {"method": "PrecisionFilter"},
         {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010},
-        {"method": "SpreadFilter", "max_spread_ratio": 0.005}
+        {"method": "SpreadFilter", "max_spread_ratio": 0.005},
+        {
+            "method": "RangeStabilityFilter",
+            "lookback_days": 10,
+            "min_rate_of_change": 0.01,
+            "refresh_period": 1440
+        }
     ],
     "exchange": {
         "name": "bittrex",

docs/includes/pairlists.md

@@ -19,6 +19,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac
 * [`PriceFilter`](#pricefilter)
 * [`ShuffleFilter`](#shufflefilter)
 * [`SpreadFilter`](#spreadfilter)
+* [`RangeStabilityFilter`](#rangestabilityfilter)
 
 !!! Tip "Testing pairlists"
     Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly.
@@ -118,6 +119,27 @@ Example:
 
 If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out.
 
+#### RangeStabilityFilter
+
+Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
+
+In the below example:
+If the trading range over the last 10 days is <1%, remove the pair from the whitelist.
+
+```json
+"pairlists": [
+    {
+        "method": "RangeStabilityFilter",
+        "lookback_days": 10,
+        "min_rate_of_change": 0.01,
+        "refresh_period": 1440
+    }
+]
+```
+
+!!! Tip
+    This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
+
 ### Full example of Pairlist Handlers
 
 The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value.
@@ -137,6 +159,12 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
     {"method": "PrecisionFilter"},
     {"method": "PriceFilter", "low_price_ratio": 0.01},
     {"method": "SpreadFilter", "max_spread_ratio": 0.005},
+    {
+        "method": "RangeStabilityFilter",
+        "lookback_days": 10,
+        "min_rate_of_change": 0.01,
+        "refresh_period": 1440
+    },
     {"method": "ShuffleFilter", "seed": 42}
     ],
 ```

freqtrade/constants.py

@@ -25,7 +25,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
                        'AgeFilter', 'PrecisionFilter', 'PriceFilter',
-                       'ShuffleFilter', 'SpreadFilter']
+                       'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000
 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'

freqtrade/exchange/exchange.py

@@ -679,12 +679,25 @@ class Exchange:
         :param pair: Pair to download
         :param timeframe: Timeframe to get data for
         :param since_ms: Timestamp in milliseconds to get history from
-        :returns List with candle (OHLCV) data
+        :return: List with candle (OHLCV) data
         """
         return asyncio.get_event_loop().run_until_complete(
             self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
                                            since_ms=since_ms))
 
+    def get_historic_ohlcv_as_df(self, pair: str, timeframe: str,
+                                 since_ms: int) -> DataFrame:
+        """
+        Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe
+        :param pair: Pair to download
+        :param timeframe: Timeframe to get data for
+        :param since_ms: Timestamp in milliseconds to get history from
+        :return: OHLCV DataFrame
+        """
+        ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms)
+        return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
+                                  drop_incomplete=self._ohlcv_partial_candle)
+
     async def _async_get_historic_ohlcv(self, pair: str,
                                         timeframe: str,
                                         since_ms: int) -> List:

freqtrade/pairlist/AgeFilter.py

@@ -49,7 +49,7 @@ class AgeFilter(IPairList):
         return (f"{self.name} - Filtering pairs with age less than "
                 f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
 
-    def _validate_pair(self, ticker: dict) -> bool:
+    def _validate_pair(self, ticker: Dict) -> bool:
         """
         Validate age for the ticker
         :param ticker: ticker dict as returned from ccxt.load_markets()

freqtrade/pairlist/rangestabilityfilter.py

@@ -0,0 +1,89 @@
+"""
+Rate of change pairlist filter
+"""
+import logging
+from typing import Any, Dict
+
+import arrow
+from cachetools.ttl import TTLCache
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.misc import plural
+from freqtrade.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class RangeStabilityFilter(IPairList):
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._days = pairlistconfig.get('lookback_days', 10)
+        self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
+        self._refresh_period = pairlistconfig.get('refresh_period', 1440)
+
+        self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period)
+
+        if self._days < 1:
+            raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1")
+        if self._days > exchange.ohlcv_candle_limit:
+            raise OperationalException("RangeStabilityFilter requires lookback_days to not "
+                                       "exceed exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit})")
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return True
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return (f"{self.name} - Filtering pairs with rate of change below "
+                f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
+
+    def _validate_pair(self, ticker: Dict) -> bool:
+        """
+        Validate trading range
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :return: True if the pair can stay, False if it should be removed
+        """
+        pair = ticker['symbol']
+        # Check symbol in cache
+        if pair in self._pair_cache:
+            return self._pair_cache[pair]
+
+        since_ms = int(arrow.utcnow()
+                       .floor('day')
+                       .shift(days=-self._days)
+                       .float_timestamp) * 1000
+
+        daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair,
+                                                                timeframe='1d',
+                                                                since_ms=since_ms)
+        result = False
+        if daily_candles is not None and not daily_candles.empty:
+            highest_high = daily_candles['high'].max()
+            lowest_low = daily_candles['low'].min()
+            pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0
+            if pct_change >= self._min_rate_of_change:
+                result = True
+            else:
+                self.log_on_refresh(logger.info,
+                                    f"Removed {pair} from whitelist, "
+                                    f"because rate of change over {plural(self._days, 'day')} is "
+                                    f"{pct_change:.3f}, which is below the "
+                                    f"threshold of {self._min_rate_of_change}.")
+                result = False
+            self._pair_cache[pair] = result
+
+        return result

tests/exchange/test_exchange.py

@@ -1307,6 +1307,57 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name):
     assert log_has_re(r"Async code raised an exception: .*", caplog)
 
 
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name):
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    ohlcv = [
+        [
+            arrow.utcnow().int_timestamp * 1000,  # unix timestamp ms
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ],
+        [
+            arrow.utcnow().shift(minutes=5).int_timestamp * 1000,  # unix timestamp ms
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ],
+        [
+            arrow.utcnow().shift(minutes=10).int_timestamp * 1000,  # unix timestamp ms
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ]
+    ]
+    pair = 'ETH/BTC'
+
+    async def mock_candle_hist(pair, timeframe, since_ms):
+        return pair, timeframe, ohlcv
+
+    exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
+    # one_call calculation * 1.8 should do 2 calls
+
+    since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8
+    ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int((
+        arrow.utcnow().int_timestamp - since) * 1000))
+
+    assert exchange._async_get_candle_history.call_count == 2
+    # Returns twice the above OHLCV data
+    assert len(ret) == 2
+    assert isinstance(ret, DataFrame)
+    assert 'date' in ret.columns
+    assert 'open' in ret.columns
+    assert 'close' in ret.columns
+    assert 'high' in ret.columns
+
+
 def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
     ohlcv = [
         [

tests/pairlist/test_pairlist.py

@@ -58,7 +58,7 @@ def whitelist_conf_2(default_conf):
 
 
 @pytest.fixture(scope="function")
-def whitelist_conf_3(default_conf):
+def whitelist_conf_agefilter(default_conf):
     default_conf['stake_currency'] = 'BTC'
     default_conf['exchange']['pair_whitelist'] = [
         'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC',
@@ -340,6 +340,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
       {"method": "PriceFilter", "low_price_ratio": 0.02}],
         "USDT", ['ETH/USDT', 'NANO/USDT']),
+    ([{"method": "StaticPairList"},
+      {"method": "RangeStabilityFilter", "lookback_days": 10,
+       "min_rate_of_change": 0.01, "refresh_period": 1440}],
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                       ohlcv_history_list, pairlists, base_currency,
@@ -528,7 +532,7 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
     assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
 
 
-def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers, caplog):
+def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers):
     default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                  {'method': 'AgeFilter', 'min_days_listed': -1}]
 
@@ -543,7 +547,7 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick
         get_patched_freqtradebot(mocker, default_conf)
 
 
-def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers, caplog):
+def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers):
     default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                  {'method': 'AgeFilter', 'min_days_listed': 99999}]
 
@@ -559,7 +563,7 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
         get_patched_freqtradebot(mocker, default_conf)
 
 
-def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_history_list):
+def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history_list):
 
     mocker.patch.multiple('freqtrade.exchange.Exchange',
                           markets=PropertyMock(return_value=markets),
@@ -571,7 +575,7 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his
         get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
     )
 
-    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_3)
+    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
     assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
     freqtrade.pairlists.refresh_pairlist()
     assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
@@ -582,6 +586,62 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his
     assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
 
 
+def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'RangeStabilityFilter', 'lookback_days': 99999}]
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+
+    with pytest.raises(OperationalException,
+                       match=r'RangeStabilityFilter requires lookback_days to not exceed '
+                             r'exchange max request size \([0-9]+\)'):
+        get_patched_freqtradebot(mocker, default_conf)
+
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'RangeStabilityFilter', 'lookback_days': 0}]
+
+    with pytest.raises(OperationalException,
+                       match='RangeStabilityFilter requires lookback_days to be >= 1'):
+        get_patched_freqtradebot(mocker, default_conf)
+
+
+@pytest.mark.parametrize('min_rate_of_change,expected_length', [
+    (0.01, 5),
+    (0.05, 0),  # Setting rate_of_change to 5% removes all pairs from the whitelist.
+])
+def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list,
+                                      min_rate_of_change, expected_length):
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'RangeStabilityFilter', 'lookback_days': 2,
+                                  'min_rate_of_change': min_rate_of_change}]
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
+    )
+
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
+    freqtrade.pairlists.refresh_pairlist()
+    assert len(freqtrade.pairlists.whitelist) == expected_length
+    assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
+
+    previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count
+    freqtrade.pairlists.refresh_pairlist()
+    assert len(freqtrade.pairlists.whitelist) == expected_length
+    # Should not have increased since first call.
+    assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
+
+
 @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [
     ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010,
       "max_price": 1.0},
@@ -617,6 +677,11 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his
      None,
      "PriceFilter requires max_price to be >= 0"
      ),  # OperationalException expected
+    ({"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01},
+     "[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below "
+     "0.01 over the last days.'}]",
+        None
+     ),
 ])
 def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
                           desc_expected, exception_expected):