adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5231 from nightshift2k/enhancement/range-based-volumefilter

Browse Source 
Range based VolumeFilter
This commit is contained in:
Matthias 2021-07-08 07:22:37 +02:00 committed by GitHub
commit 225522762b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 234 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
docs/includes/pairlists.md
View File

@ -64,17 +64,56 @@ The `refresh_period` setting allows to define the period (in seconds), at which
The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
`VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
* The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours.
```json
"pairlists": [{
"pairlists": [
{
"method": "VolumePairList",
"number_assets": 20,
"sort_key": "quoteVolume",
"refresh_period": 1800
}],
}
],
```
`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
```json
"pairlists": [
{
"method": "VolumePairList",
"number_assets": 20,
"sort_key": "quoteVolume",
"refresh_period": 86400,
"lookback_days": 7
}
],
```
!!! Warning "Range look back and refresh period"
When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
!!! Warning "Performance implications when using lookback range"
If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
```json
"pairlists": [
{
"method": "VolumePairList",
"number_assets": 20,
"sort_key": "quoteVolume",
"refresh_period": 3600,
"lookback_timeframe": "1h",
"lookback_period": 72
}
],
```
!!! Note

94
freqtrade/plugins/pairlist/VolumePairList.py
View File

@ -6,9 +6,12 @@ Provides dynamic pair list based on trade volumes
import logging
from typing import Any, Dict, List
import arrow
from cachetools.ttl import TTLCache
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import format_ms_time
from freqtrade.plugins.pairlist.IPairList import IPairList
@ -36,6 +39,35 @@ class VolumePairList(IPairList):
self._min_value = self._pairlistconfig.get('min_value', 0)
self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
if (self._lookback_days > 0) & (self._lookback_period > 0):
raise OperationalException(
'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
'config. Please set lookback_days only or lookback_period and lookback_timeframe '
'and restart the bot.'
)
# overwrite lookback timeframe and days when lookback_days is set
if self._lookback_days > 0:
self._lookback_timeframe = '1d'
self._lookback_period = self._lookback_days
# get timeframe in minutes and seconds
self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
self._tf_in_sec = self._tf_in_min * 60
# wether to use range lookback or not
self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
if self._use_range & (self._refresh_period < self._tf_in_sec):
raise OperationalException(
f'Refresh period of {self._refresh_period} seconds is smaller than one '
f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
f'to at least {self._tf_in_sec} and restart the bot.'
)
if not self._exchange.exchange_has('fetchTickers'):
raise OperationalException(
@ -47,6 +79,13 @@ class VolumePairList(IPairList):
raise OperationalException(
f'key {self._sort_key} not in {SORT_VALUES}')
if self._lookback_period < 0:
raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
raise OperationalException("VolumeFilter requires lookback_period to not "
"exceed exchange max request size "
f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
@property
def needstickers(self) -> bool:
"""
@ -78,7 +117,6 @@ class VolumePairList(IPairList):
# Item found - no refresh necessary
return pairlist
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
filtered_tickers = [
@ -103,6 +141,60 @@ class VolumePairList(IPairList):
# Use the incoming pairlist.
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
if self._use_range:
since_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-(self._lookback_period * self._tf_in_min)
- self._tf_in_min)
.int_timestamp) * 1000
to_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-self._tf_in_min)
.int_timestamp) * 1000
# todo: utc date output for starting date
self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
f"till {format_ms_time(to_ms)}", logger.info)
needed_pairs = [
(p, self._lookback_timeframe) for p in
[
s['symbol'] for s in filtered_tickers
] if p not in self._pair_cache
]
# Get all candles
candles = {}
if needed_pairs:
candles = self._exchange.refresh_latest_ohlcv(
needed_pairs, since_ms=since_ms, cache=False
)
for i, p in enumerate(filtered_tickers):
pair_candles = candles[
(p['symbol'], self._lookback_timeframe)
] if (p['symbol'], self._lookback_timeframe) in candles else None
# in case of candle data calculate typical price and quoteVolume for candle
if pair_candles is not None and not pair_candles.empty:
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3
pair_candles['quoteVolume'] = (
pair_candles['volume'] * pair_candles['typical_price']
)
# ensure that a rolling sum over the lookback_period is built
# if pair_candles contains more candles than lookback_period
quoteVolume = (pair_candles['quoteVolume']
.rolling(self._lookback_period)
.sum()
.iloc[-1])
# replace quoteVolume with range quoteVolume sum calculated above
filtered_tickers[i]['quoteVolume'] = quoteVolume
else:
filtered_tickers[i]['quoteVolume'] = 0
if self._min_value > 0:
filtered_tickers = [
v for v in filtered_tickers if v[self._sort_key] > self._min_value]

99
tests/plugins/test_pairlist.py
View File

@ -536,6 +536,105 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog)
@pytest.mark.parametrize("pairlists,base_currency,volumefilter_result", [
# default refresh of 1800 to small for daily candle lookback
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
"lookback_days": 1}],
"BTC", "default_refresh_too_short"), # OperationalException expected
# ambigous configuration with lookback days and period
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
"lookback_days": 1, "lookback_period": 1}],
"BTC", "lookback_days_and_period"), # OperationalException expected
# negative lookback period
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
"lookback_timeframe": "1d", "lookback_period": -1}],
"BTC", "lookback_period_negative"), # OperationalException expected
# lookback range exceedes exchange limit
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
"lookback_timeframe": "1m", "lookback_period": 2000, "refresh_period": 3600}],
"BTC", 'lookback_exceeds_exchange_request_size'), # OperationalException expected
# expecing pairs as given
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
"lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
"BTC", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']),
# expecting pairs from default tickers, because 1h candles are not available
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
"lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}],
"BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']),
])
def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history,
pairlists, base_currency, volumefilter_result, caplog) -> None:
whitelist_conf['pairlists'] = pairlists
whitelist_conf['stake_currency'] = base_currency
ohlcv_history_high_vola = ohlcv_history.copy()
ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090
# create candles for medium overall volume with last candle high volume
ohlcv_history_medium_volume = ohlcv_history.copy()
ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5
# create candles for high volume with all candles high volume
ohlcv_history_high_volume = ohlcv_history.copy()
ohlcv_history_high_volume.loc[:, 'volume'] = 10
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history_medium_volume,
('XRP/BTC', '1d'): ohlcv_history_high_vola,
('HOT/BTC', '1d'): ohlcv_history_high_volume,
}
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
if volumefilter_result == 'default_refresh_too_short':
with pytest.raises(OperationalException,
match=r'Refresh period of [0-9]+ seconds is smaller than one timeframe '
r'of [0-9]+.*\. Please adjust refresh_period to at least [0-9]+ '
r'and restart the bot\.'):
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
return
elif volumefilter_result == 'lookback_days_and_period':
with pytest.raises(OperationalException,
match=r'Ambigous configuration: lookback_days and lookback_period both '
r'set in pairlist config\..*'):
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
elif volumefilter_result == 'lookback_period_negative':
with pytest.raises(OperationalException,
match=r'VolumeFilter requires lookback_period to be >= 0'):
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
elif volumefilter_result == 'lookback_exceeds_exchange_request_size':
with pytest.raises(OperationalException,
match=r'VolumeFilter requires lookback_period to not exceed '
r'exchange max request size \([0-9]+\)'):
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
else:
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_tickers=tickers,
markets=PropertyMock(return_value=shitcoinmarkets)
)
# remove ohlcv when looback_timeframe != 1d
# to enforce fallback to ticker data
if 'lookback_timeframe' in pairlists[0]:
if pairlists[0]['lookback_timeframe'] != '1d':
ohlcv_data = []
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
)
freqtrade.pairlists.refresh_pairlist()
whitelist = freqtrade.pairlists.whitelist
assert isinstance(whitelist, list)
assert whitelist == volumefilter_result
def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}]
del whitelist_conf['stoploss']