Merge pull request #5231 from nightshift2k/enhancement/range-based-volumefilter

Range based VolumeFilter

docs/includes/pairlists.md

@@ -64,17 +64,56 @@ The `refresh_period` setting allows to define the period (in seconds), at which
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
 Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
 
-`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
+`VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
 * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours.
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "refresh_period": 1800
-}],
+    }
+],
+```
+
+`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
+
+For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
+
+```json
+"pairlists": [
+    {
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "refresh_period": 86400,
+        "lookback_days": 7
+    }
+],
+```
+
+!!! Warning "Range look back and refresh period"
+    When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
+
+!!! Warning "Performance implications when using lookback range"
+    If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
+
+More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
+
+```json
+"pairlists": [
+    {
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "refresh_period": 3600,
+        "lookback_timeframe": "1h",
+        "lookback_period": 72
+    }
+],
 ```
 
 !!! Note

freqtrade/plugins/pairlist/VolumePairList.py

@@ -6,9 +6,12 @@ Provides dynamic pair list based on trade volumes
 import logging
 from typing import Any, Dict, List
 
+import arrow
 from cachetools.ttl import TTLCache
 
 from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.misc import format_ms_time
 from freqtrade.plugins.pairlist.IPairList import IPairList
 
 
@@ -36,6 +39,35 @@ class VolumePairList(IPairList):
         self._min_value = self._pairlistconfig.get('min_value', 0)
         self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
         self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
+        self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
+        self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
+
+        if (self._lookback_days > 0) & (self._lookback_period > 0):
+            raise OperationalException(
+                'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
+                'config. Please set lookback_days only or lookback_period and lookback_timeframe '
+                'and restart the bot.'
+            )
+
+        # overwrite lookback timeframe and days when lookback_days is set
+        if self._lookback_days > 0:
+            self._lookback_timeframe = '1d'
+            self._lookback_period = self._lookback_days
+
+        # get timeframe in minutes and seconds
+        self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
+        self._tf_in_sec = self._tf_in_min * 60
+
+        # wether to use range lookback or not
+        self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
+
+        if self._use_range & (self._refresh_period < self._tf_in_sec):
+            raise OperationalException(
+                f'Refresh period of {self._refresh_period} seconds is smaller than one '
+                f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
+                f'to at least {self._tf_in_sec} and restart the bot.'
+            )
 
         if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -47,6 +79,13 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
+        if self._lookback_period < 0:
+            raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
+        if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
+            raise OperationalException("VolumeFilter requires lookback_period to not "
+                                       "exceed exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
+
     @property
     def needstickers(self) -> bool:
         """
@@ -78,7 +117,6 @@ class VolumePairList(IPairList):
             # Item found - no refresh necessary
             return pairlist
         else:
-
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
             filtered_tickers = [
@@ -103,6 +141,60 @@ class VolumePairList(IPairList):
         # Use the incoming pairlist.
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
+        # get lookback period in ms, for exchange ohlcv fetch
+        if self._use_range:
+            since_ms = int(arrow.utcnow()
+                           .floor('minute')
+                           .shift(minutes=-(self._lookback_period * self._tf_in_min)
+                                  - self._tf_in_min)
+                           .int_timestamp) * 1000
+
+            to_ms = int(arrow.utcnow()
+                        .floor('minute')
+                        .shift(minutes=-self._tf_in_min)
+                        .int_timestamp) * 1000
+
+            # todo: utc date output for starting date
+            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
+                          f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
+                          f"till {format_ms_time(to_ms)}", logger.info)
+            needed_pairs = [
+                (p, self._lookback_timeframe) for p in
+                [
+                    s['symbol'] for s in filtered_tickers
+                ] if p not in self._pair_cache
+            ]
+
+            # Get all candles
+            candles = {}
+            if needed_pairs:
+                candles = self._exchange.refresh_latest_ohlcv(
+                    needed_pairs, since_ms=since_ms, cache=False
+                )
+            for i, p in enumerate(filtered_tickers):
+                pair_candles = candles[
+                    (p['symbol'], self._lookback_timeframe)
+                ] if (p['symbol'], self._lookback_timeframe) in candles else None
+                # in case of candle data calculate typical price and quoteVolume for candle
+                if pair_candles is not None and not pair_candles.empty:
+                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+                                                     + pair_candles['close']) / 3
+                    pair_candles['quoteVolume'] = (
+                        pair_candles['volume'] * pair_candles['typical_price']
+                    )
+
+                    # ensure that a rolling sum over the lookback_period is built
+                    # if pair_candles contains more candles than lookback_period
+                    quoteVolume = (pair_candles['quoteVolume']
+                                   .rolling(self._lookback_period)
+                                   .sum()
+                                   .iloc[-1])
+
+                    # replace quoteVolume with range quoteVolume sum calculated above
+                    filtered_tickers[i]['quoteVolume'] = quoteVolume
+                else:
+                    filtered_tickers[i]['quoteVolume'] = 0
+
         if self._min_value > 0:
             filtered_tickers = [
                     v for v in filtered_tickers if v[self._sort_key] > self._min_value]

tests/plugins/test_pairlist.py

@@ -536,6 +536,105 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
                 assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog)
 
 
+@pytest.mark.parametrize("pairlists,base_currency,volumefilter_result", [
+    # default refresh of 1800 to small for daily candle lookback
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_days": 1}],
+     "BTC", "default_refresh_too_short"),  # OperationalException expected
+    # ambigous configuration with lookback days and period
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_days": 1, "lookback_period": 1}],
+     "BTC", "lookback_days_and_period"),  # OperationalException expected
+    # negative lookback period
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1d", "lookback_period": -1}],
+     "BTC", "lookback_period_negative"),  # OperationalException expected
+    # lookback range exceedes exchange limit
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1m", "lookback_period": 2000, "refresh_period": 3600}],
+     "BTC", 'lookback_exceeds_exchange_request_size'),  # OperationalException expected
+    # expecing pairs as given
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
+     "BTC", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']),
+    # expecting pairs from default tickers, because 1h candles are not available
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}],
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']),
+])
+def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history,
+                              pairlists, base_currency, volumefilter_result, caplog) -> None:
+    whitelist_conf['pairlists'] = pairlists
+    whitelist_conf['stake_currency'] = base_currency
+
+    ohlcv_history_high_vola = ohlcv_history.copy()
+    ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090
+
+    # create candles for medium overall volume with last candle high volume
+    ohlcv_history_medium_volume = ohlcv_history.copy()
+    ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5
+
+    # create candles for high volume with all candles high volume
+    ohlcv_history_high_volume = ohlcv_history.copy()
+    ohlcv_history_high_volume.loc[:, 'volume'] = 10
+
+    ohlcv_data = {
+        ('ETH/BTC', '1d'): ohlcv_history,
+        ('TKN/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history_medium_volume,
+        ('XRP/BTC', '1d'): ohlcv_history_high_vola,
+        ('HOT/BTC', '1d'): ohlcv_history_high_volume,
+    }
+
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    if volumefilter_result == 'default_refresh_too_short':
+        with pytest.raises(OperationalException,
+                           match=r'Refresh period of [0-9]+ seconds is smaller than one timeframe '
+                                 r'of [0-9]+.*\. Please adjust refresh_period to at least [0-9]+ '
+                                 r'and restart the bot\.'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+        return
+    elif volumefilter_result == 'lookback_days_and_period':
+        with pytest.raises(OperationalException,
+                           match=r'Ambigous configuration: lookback_days and lookback_period both '
+                                 r'set in pairlist config\..*'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    elif volumefilter_result == 'lookback_period_negative':
+        with pytest.raises(OperationalException,
+                           match=r'VolumeFilter requires lookback_period to be >= 0'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    elif volumefilter_result == 'lookback_exceeds_exchange_request_size':
+        with pytest.raises(OperationalException,
+                           match=r'VolumeFilter requires lookback_period to not exceed '
+                           r'exchange max request size \([0-9]+\)'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    else:
+        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            get_tickers=tickers,
+            markets=PropertyMock(return_value=shitcoinmarkets)
+        )
+
+        # remove ohlcv when looback_timeframe != 1d
+        # to enforce fallback to ticker data
+        if 'lookback_timeframe' in pairlists[0]:
+            if pairlists[0]['lookback_timeframe'] != '1d':
+                ohlcv_data = []
+
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
+        )
+
+        freqtrade.pairlists.refresh_pairlist()
+        whitelist = freqtrade.pairlists.whitelist
+
+        assert isinstance(whitelist, list)
+        assert whitelist == volumefilter_result
+
+
 def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
     whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}]
     del whitelist_conf['stoploss']