Merge pull request #4686 from gerrito333/develop

Add Pairlist Handler for Volatility

docs/includes/pairlists.md

@@ -4,7 +4,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade.
 
 In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler).
 
-Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
+Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
 
 If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
 
@@ -29,6 +29,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`ShuffleFilter`](#shufflefilter)
 * [`SpreadFilter`](#spreadfilter)
 * [`RangeStabilityFilter`](#rangestabilityfilter)
+* [`VolatilityFilter`](#volatilityfilter)
 
 !!! Tip "Testing pairlists"
     Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly.
@@ -164,6 +165,29 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit
 !!! Tip
     This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
 
+#### VolatilityFilter
+
+Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatility of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)).
+
+This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
+
+This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. 
+
+In the below example:
+If the volatility over the last 10 days is not in the range of 0.20-0.30, remove the pair from the whitelist. The filter is applied every 24h.
+
+```json
+"pairlists": [
+    {
+        "method": "VolatilityFilter",
+        "lookback_days": 10,
+        "min_volatility": 0.05,
+        "max_volatility": 0.50,
+        "refresh_period": 86400
+    }
+]
+```
+
 ### Full example of Pairlist Handlers
 
 The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value.
@@ -189,6 +213,13 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
         "min_rate_of_change": 0.01,
         "refresh_period": 1440
     },
+    {
+        "method": "VolatilityFilter",
+        "lookback_days": 10,
+        "min_volatility": 0.05,
+        "max_volatility": 0.50,
+        "refresh_period": 86400
+    },
     {"method": "ShuffleFilter", "seed": 42}
     ],
 ```

freqtrade/constants.py

@@ -26,7 +26,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
                        'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
                        'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
-                       'SpreadFilter']
+                       'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -0,0 +1,121 @@
+"""
+Volatility pairlist filter
+"""
+import logging
+import sys
+from copy import deepcopy
+from typing import Any, Dict, List, Optional
+
+import arrow
+import numpy as np
+from cachetools.ttl import TTLCache
+from pandas import DataFrame
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.misc import plural
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class VolatilityFilter(IPairList):
+    '''
+    Filters pairs by volatility
+    '''
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._days = pairlistconfig.get('lookback_days', 10)
+        self._min_volatility = pairlistconfig.get('min_volatility', 0)
+        self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize)
+        self._refresh_period = pairlistconfig.get('refresh_period', 1440)
+
+        self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
+
+        if self._days < 1:
+            raise OperationalException("VolatilityFilter requires lookback_days to be >= 1")
+        if self._days > exchange.ohlcv_candle_limit('1d'):
+            raise OperationalException("VolatilityFilter requires lookback_days to not "
+                                       "exceed exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit('1d')})")
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return (f"{self.name} - Filtering pairs with volatility range "
+                f"{self._min_volatility}-{self._max_volatility} "
+                f" the last {self._days} {plural(self._days, 'day')}.")
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Validate trading range
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new allowlist
+        """
+        needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
+
+        since_ms = int(arrow.utcnow()
+                       .floor('day')
+                       .shift(days=-self._days - 1)
+                       .float_timestamp) * 1000
+        # Get all candles
+        candles = {}
+        if needed_pairs:
+            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
+                                                          cache=False)
+
+        if self._enabled:
+            for p in deepcopy(pairlist):
+                daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
+                if not self._validate_pair_loc(p, daily_candles):
+                    pairlist.remove(p)
+        return pairlist
+
+    def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
+        """
+        Validate trading range
+        :param pair: Pair that's currently validated
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :return: True if the pair can stay, false if it should be removed
+        """
+        # Check symbol in cache
+        cached_res = self._pair_cache.get(pair, None)
+        if cached_res is not None:
+            return cached_res
+
+        result = False
+        if daily_candles is not None and not daily_candles.empty:
+            returns = (np.log(daily_candles.close / daily_candles.close.shift(-1)))
+            returns.fillna(0, inplace=True)
+
+            volatility_series = returns.rolling(window=self._days).std()*np.sqrt(self._days)
+            volatility_avg = volatility_series.mean()
+
+            if self._min_volatility <= volatility_avg <= self._max_volatility:
+                result = True
+            else:
+                self.log_once(f"Removed {pair} from whitelist, because volatility "
+                              f"over {self._days} {plural(self._days, 'day')} "
+                              f"is: {volatility_avg:.3f} "
+                              f"which is not in the configured range of "
+                              f"{self._min_volatility}-{self._max_volatility}.",
+                              logger.info)
+                result = False
+            self._pair_cache[pair] = result
+
+        return result

tests/plugins/test_pairlist.py

@@ -407,6 +407,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
       {"method": "RangeStabilityFilter", "lookback_days": 10,
        "min_rate_of_change": 0.01, "refresh_period": 1440}],
      "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
+    ([{"method": "StaticPairList"},
+      {"method": "VolatilityFilter", "lookback_days": 3,
+       "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'])
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                       ohlcv_history, pairlists, base_currency,
@@ -414,12 +418,15 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
     whitelist_conf['pairlists'] = pairlists
     whitelist_conf['stake_currency'] = base_currency
 
+    ohlcv_history_high_vola = ohlcv_history.copy()
+    ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090
+
     ohlcv_data = {
         ('ETH/BTC', '1d'): ohlcv_history,
         ('TKN/BTC', '1d'): ohlcv_history,
         ('LTC/BTC', '1d'): ohlcv_history,
         ('XRP/BTC', '1d'): ohlcv_history,
-        ('HOT/BTC', '1d'): ohlcv_history,
+        ('HOT/BTC', '1d'): ohlcv_history_high_vola,
     }
 
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
@@ -487,6 +494,8 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
                     assert log_has(logmsg, caplog)
                 else:
                     assert not log_has(logmsg, caplog)
+            if pairlist["method"] == 'VolatilityFilter':
+                assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog)
 
 
 def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: