Merge pull request #3975 from freqtrade/volatility_filter

RangeStabilityFilter filter - filter for pairs without much movement
This commit is contained in:
Matthias 2020-11-25 19:17:06 +01:00 committed by GitHub
commit 53231d94a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 261 additions and 9 deletions

View File

@ -67,7 +67,13 @@
{"method": "AgeFilter", "min_days_listed": 10}, {"method": "AgeFilter", "min_days_listed": 10},
{"method": "PrecisionFilter"}, {"method": "PrecisionFilter"},
{"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010}, {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010},
{"method": "SpreadFilter", "max_spread_ratio": 0.005} {"method": "SpreadFilter", "max_spread_ratio": 0.005},
{
"method": "RangeStabilityFilter",
"lookback_days": 10,
"min_rate_of_change": 0.01,
"refresh_period": 1440
}
], ],
"exchange": { "exchange": {
"name": "bittrex", "name": "bittrex",

View File

@ -19,6 +19,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac
* [`PriceFilter`](#pricefilter) * [`PriceFilter`](#pricefilter)
* [`ShuffleFilter`](#shufflefilter) * [`ShuffleFilter`](#shufflefilter)
* [`SpreadFilter`](#spreadfilter) * [`SpreadFilter`](#spreadfilter)
* [`RangeStabilityFilter`](#rangestabilityfilter)
!!! Tip "Testing pairlists" !!! Tip "Testing pairlists"
Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly. Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly.
@ -118,6 +119,27 @@ Example:
If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out. If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out.
#### RangeStabilityFilter
Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
In the below example:
If the trading range over the last 10 days is <1%, remove the pair from the whitelist.
```json
"pairlists": [
{
"method": "RangeStabilityFilter",
"lookback_days": 10,
"min_rate_of_change": 0.01,
"refresh_period": 1440
}
]
```
!!! Tip
This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
### Full example of Pairlist Handlers ### Full example of Pairlist Handlers
The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value.
@ -137,6 +159,12 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
{"method": "PrecisionFilter"}, {"method": "PrecisionFilter"},
{"method": "PriceFilter", "low_price_ratio": 0.01}, {"method": "PriceFilter", "low_price_ratio": 0.01},
{"method": "SpreadFilter", "max_spread_ratio": 0.005}, {"method": "SpreadFilter", "max_spread_ratio": 0.005},
{
"method": "RangeStabilityFilter",
"lookback_days": 10,
"min_rate_of_change": 0.01,
"refresh_period": 1440
},
{"method": "ShuffleFilter", "seed": 42} {"method": "ShuffleFilter", "seed": 42}
], ],
``` ```

View File

@ -25,7 +25,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
'AgeFilter', 'PrecisionFilter', 'PriceFilter', 'AgeFilter', 'PrecisionFilter', 'PriceFilter',
'ShuffleFilter', 'SpreadFilter'] 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
DRY_RUN_WALLET = 1000 DRY_RUN_WALLET = 1000
DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'

View File

@ -679,12 +679,25 @@ class Exchange:
:param pair: Pair to download :param pair: Pair to download
:param timeframe: Timeframe to get data for :param timeframe: Timeframe to get data for
:param since_ms: Timestamp in milliseconds to get history from :param since_ms: Timestamp in milliseconds to get history from
:returns List with candle (OHLCV) data :return: List with candle (OHLCV) data
""" """
return asyncio.get_event_loop().run_until_complete( return asyncio.get_event_loop().run_until_complete(
self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
since_ms=since_ms)) since_ms=since_ms))
def get_historic_ohlcv_as_df(self, pair: str, timeframe: str,
since_ms: int) -> DataFrame:
"""
Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe
:param pair: Pair to download
:param timeframe: Timeframe to get data for
:param since_ms: Timestamp in milliseconds to get history from
:return: OHLCV DataFrame
"""
ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms)
return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
async def _async_get_historic_ohlcv(self, pair: str, async def _async_get_historic_ohlcv(self, pair: str,
timeframe: str, timeframe: str,
since_ms: int) -> List: since_ms: int) -> List:

View File

@ -49,7 +49,7 @@ class AgeFilter(IPairList):
return (f"{self.name} - Filtering pairs with age less than " return (f"{self.name} - Filtering pairs with age less than "
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
def _validate_pair(self, ticker: dict) -> bool: def _validate_pair(self, ticker: Dict) -> bool:
""" """
Validate age for the ticker Validate age for the ticker
:param ticker: ticker dict as returned from ccxt.load_markets() :param ticker: ticker dict as returned from ccxt.load_markets()

View File

@ -0,0 +1,89 @@
"""
Rate of change pairlist filter
"""
import logging
from typing import Any, Dict
import arrow
from cachetools.ttl import TTLCache
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class RangeStabilityFilter(IPairList):
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._days = pairlistconfig.get('lookback_days', 10)
self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period)
if self._days < 1:
raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1")
if self._days > exchange.ohlcv_candle_limit:
raise OperationalException("RangeStabilityFilter requires lookback_days to not "
"exceed exchange max request size "
f"({exchange.ohlcv_candle_limit})")
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return True
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - Filtering pairs with rate of change below "
f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
def _validate_pair(self, ticker: Dict) -> bool:
"""
Validate trading range
:param ticker: ticker dict as returned from ccxt.load_markets()
:return: True if the pair can stay, False if it should be removed
"""
pair = ticker['symbol']
# Check symbol in cache
if pair in self._pair_cache:
return self._pair_cache[pair]
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._days)
.float_timestamp) * 1000
daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair,
timeframe='1d',
since_ms=since_ms)
result = False
if daily_candles is not None and not daily_candles.empty:
highest_high = daily_candles['high'].max()
lowest_low = daily_candles['low'].min()
pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0
if pct_change >= self._min_rate_of_change:
result = True
else:
self.log_on_refresh(logger.info,
f"Removed {pair} from whitelist, "
f"because rate of change over {plural(self._days, 'day')} is "
f"{pct_change:.3f}, which is below the "
f"threshold of {self._min_rate_of_change}.")
result = False
self._pair_cache[pair] = result
return result

View File

@ -1307,6 +1307,57 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name):
assert log_has_re(r"Async code raised an exception: .*", caplog) assert log_has_re(r"Async code raised an exception: .*", caplog)
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name):
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
ohlcv = [
[
arrow.utcnow().int_timestamp * 1000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
],
[
arrow.utcnow().shift(minutes=5).int_timestamp * 1000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
],
[
arrow.utcnow().shift(minutes=10).int_timestamp * 1000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
]
]
pair = 'ETH/BTC'
async def mock_candle_hist(pair, timeframe, since_ms):
return pair, timeframe, ohlcv
exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
# one_call calculation * 1.8 should do 2 calls
since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8
ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int((
arrow.utcnow().int_timestamp - since) * 1000))
assert exchange._async_get_candle_history.call_count == 2
# Returns twice the above OHLCV data
assert len(ret) == 2
assert isinstance(ret, DataFrame)
assert 'date' in ret.columns
assert 'open' in ret.columns
assert 'close' in ret.columns
assert 'high' in ret.columns
def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
ohlcv = [ ohlcv = [
[ [

View File

@ -58,7 +58,7 @@ def whitelist_conf_2(default_conf):
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def whitelist_conf_3(default_conf): def whitelist_conf_agefilter(default_conf):
default_conf['stake_currency'] = 'BTC' default_conf['stake_currency'] = 'BTC'
default_conf['exchange']['pair_whitelist'] = [ default_conf['exchange']['pair_whitelist'] = [
'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC',
@ -340,6 +340,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
{"method": "PriceFilter", "low_price_ratio": 0.02}], {"method": "PriceFilter", "low_price_ratio": 0.02}],
"USDT", ['ETH/USDT', 'NANO/USDT']), "USDT", ['ETH/USDT', 'NANO/USDT']),
([{"method": "StaticPairList"},
{"method": "RangeStabilityFilter", "lookback_days": 10,
"min_rate_of_change": 0.01, "refresh_period": 1440}],
"BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
]) ])
def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
ohlcv_history_list, pairlists, base_currency, ohlcv_history_list, pairlists, base_currency,
@ -528,7 +532,7 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers, caplog): def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers):
default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
{'method': 'AgeFilter', 'min_days_listed': -1}] {'method': 'AgeFilter', 'min_days_listed': -1}]
@ -543,7 +547,7 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick
get_patched_freqtradebot(mocker, default_conf) get_patched_freqtradebot(mocker, default_conf)
def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers, caplog): def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers):
default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
{'method': 'AgeFilter', 'min_days_listed': 99999}] {'method': 'AgeFilter', 'min_days_listed': 99999}]
@ -559,7 +563,7 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
get_patched_freqtradebot(mocker, default_conf) get_patched_freqtradebot(mocker, default_conf)
def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_history_list): def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history_list):
mocker.patch.multiple('freqtrade.exchange.Exchange', mocker.patch.multiple('freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets), markets=PropertyMock(return_value=markets),
@ -571,7 +575,7 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his
get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
) )
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_3) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
assert freqtrade.exchange.get_historic_ohlcv.call_count == 0 assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
freqtrade.pairlists.refresh_pairlist() freqtrade.pairlists.refresh_pairlist()
assert freqtrade.exchange.get_historic_ohlcv.call_count > 0 assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
@ -582,6 +586,62 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his
assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
{'method': 'RangeStabilityFilter', 'lookback_days': 99999}]
mocker.patch.multiple('freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets),
exchange_has=MagicMock(return_value=True),
get_tickers=tickers
)
with pytest.raises(OperationalException,
match=r'RangeStabilityFilter requires lookback_days to not exceed '
r'exchange max request size \([0-9]+\)'):
get_patched_freqtradebot(mocker, default_conf)
default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
{'method': 'RangeStabilityFilter', 'lookback_days': 0}]
with pytest.raises(OperationalException,
match='RangeStabilityFilter requires lookback_days to be >= 1'):
get_patched_freqtradebot(mocker, default_conf)
@pytest.mark.parametrize('min_rate_of_change,expected_length', [
(0.01, 5),
(0.05, 0), # Setting rate_of_change to 5% removes all pairs from the whitelist.
])
def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list,
min_rate_of_change, expected_length):
default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
{'method': 'RangeStabilityFilter', 'lookback_days': 2,
'min_rate_of_change': min_rate_of_change}]
mocker.patch.multiple('freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets),
exchange_has=MagicMock(return_value=True),
get_tickers=tickers
)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
)
freqtrade = get_patched_freqtradebot(mocker, default_conf)
assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == expected_length
assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == expected_length
# Should not have increased since first call.
assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
@pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [ @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [
({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010, ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010,
"max_price": 1.0}, "max_price": 1.0},
@ -617,6 +677,11 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his
None, None,
"PriceFilter requires max_price to be >= 0" "PriceFilter requires max_price to be >= 0"
), # OperationalException expected ), # OperationalException expected
({"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01},
"[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below "
"0.01 over the last days.'}]",
None
),
]) ])
def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
desc_expected, exception_expected): desc_expected, exception_expected):