Merge pull request #3521 from gambcl/develop

AgeFilter for filtering out newly listed pairs
This commit is contained in:
Matthias 2020-06-25 09:56:34 +02:00 committed by GitHub
commit c10545ef89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 171 additions and 13 deletions

View File

@ -64,6 +64,7 @@
"sort_key": "quoteVolume",
"refresh_period": 1800
},
{"method": "AgeFilter", "min_days_listed": 10},
{"method": "PrecisionFilter"},
{"method": "PriceFilter", "low_price_ratio": 0.01},
{"method": "SpreadFilter", "max_spread_ratio": 0.005}

View File

@ -592,7 +592,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade.
In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler).
Additionaly, [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
Additionaly, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
@ -602,6 +602,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac
* [`StaticPairList`](#static-pair-list) (default, if not configured differently)
* [`VolumePairList`](#volume-pair-list)
* [`AgeFilter`](#agefilter)
* [`PrecisionFilter`](#precisionfilter)
* [`PriceFilter`](#pricefilter)
* [`ShuffleFilter`](#shufflefilter)
@ -645,6 +646,16 @@ The `refresh_period` setting allows to define the period (in seconds), at which
}],
```
#### AgeFilter
Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`).
When pairs are first listed on an exchange they can suffer huge price drops and volatility
in the first few days while the pair goes through its price-discovery period. Bots can often
be caught out buying before the pair has finished dropping in price.
This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days.
#### PrecisionFilter
Filters low-value coins which would not allow setting stoplosses.
@ -692,6 +703,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
"number_assets": 20,
"sort_key": "quoteVolume",
},
{"method": "AgeFilter", "min_days_listed": 10},
{"method": "PrecisionFilter"},
{"method": "PriceFilter", "low_price_ratio": 0.01},
{"method": "SpreadFilter", "max_spread_ratio": 0.005},

View File

@ -22,7 +22,8 @@ ORDERBOOK_SIDES = ['ask', 'bid']
ORDERTYPE_POSSIBILITIES = ['limit', 'market']
ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
'PrecisionFilter', 'PriceFilter', 'ShuffleFilter', 'SpreadFilter']
'AgeFilter', 'PrecisionFilter', 'PriceFilter',
'ShuffleFilter', 'SpreadFilter']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
DRY_RUN_WALLET = 1000
MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons

View File

@ -0,0 +1,76 @@
"""
Minimum age (days listed) pair list filter
"""
import logging
import arrow
from typing import Any, Dict
from freqtrade.misc import plural
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class AgeFilter(IPairList):
# Checked symbols cache (dictionary of ticker symbol => timestamp)
_symbolsChecked: Dict[str, int] = {}
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
self._enabled = self._min_days_listed >= 1
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return True
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - Filtering pairs with age less than "
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
def _validate_pair(self, ticker: dict) -> bool:
"""
Validate age for the ticker
:param ticker: ticker dict as returned from ccxt.load_markets()
:return: True if the pair can stay, False if it should be removed
"""
# Check symbol in cache
if ticker['symbol'] in self._symbolsChecked:
return True
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._min_days_listed)
.float_timestamp) * 1000
daily_candles = self._exchange.get_historic_ohlcv(pair=ticker['symbol'],
timeframe='1d',
since_ms=since_ms)
if daily_candles is not None:
if len(daily_candles) > self._min_days_listed:
# We have fetched at least the minimum required number of daily candles
# Add to cache, store the time we last checked this symbol
self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000
return True
else:
self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because age is less than "
f"{self._min_days_listed} "
f"{plural(self._min_days_listed, 'day')}")
return False
return False

View File

@ -68,7 +68,7 @@ class IPairList(ABC):
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""

View File

@ -27,7 +27,7 @@ class PrecisionFilter(IPairList):
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return True

View File

@ -24,7 +24,7 @@ class PriceFilter(IPairList):
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return True

View File

@ -25,7 +25,7 @@ class ShuffleFilter(IPairList):
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return False

View File

@ -24,7 +24,7 @@ class SpreadFilter(IPairList):
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return True

View File

@ -28,7 +28,7 @@ class StaticPairList(IPairList):
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return False

View File

@ -54,7 +54,7 @@ class VolumePairList(IPairList):
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return True

View File

@ -57,6 +57,31 @@ def whitelist_conf_2(default_conf):
return default_conf
@pytest.fixture(scope="function")
def whitelist_conf_3(default_conf):
default_conf['stake_currency'] = 'BTC'
default_conf['exchange']['pair_whitelist'] = [
'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC',
'BTT/BTC', 'HOT/BTC', 'FUEL/BTC', 'XRP/BTC'
]
default_conf['exchange']['pair_blacklist'] = [
'BLK/BTC'
]
default_conf['pairlists'] = [
{
"method": "VolumePairList",
"number_assets": 5,
"sort_key": "quoteVolume",
"refresh_period": 0,
},
{
"method": "AgeFilter",
"min_days_listed": 2
}
]
return default_conf
@pytest.fixture(scope="function")
def static_pl_conf(whitelist_conf):
whitelist_conf['pairlists'] = [
@ -220,11 +245,20 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
# No pair for ETH, all handlers
([{"method": "StaticPairList"},
{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
{"method": "AgeFilter", "min_days_listed": 2},
{"method": "PrecisionFilter"},
{"method": "PriceFilter", "low_price_ratio": 0.03},
{"method": "SpreadFilter", "max_spread_ratio": 0.005},
{"method": "ShuffleFilter"}],
"ETH", []),
# AgeFilter and VolumePairList (require 2 days only, all should pass age test)
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
{"method": "AgeFilter", "min_days_listed": 2}],
"BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
# AgeFilter and VolumePairList (require 10 days, all should fail age test)
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
{"method": "AgeFilter", "min_days_listed": 10}],
"BTC", []),
# Precisionfilter and quote volume
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
{"method": "PrecisionFilter"}],
@ -272,7 +306,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
# ShuffleFilter, no seed
([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
{"method": "ShuffleFilter"}],
"USDT", 3), # whitelist_result is integer -- check only lenght of randomized pairlist
"USDT", 3), # whitelist_result is integer -- check only length of randomized pairlist
# AgeFilter only
([{"method": "AgeFilter", "min_days_listed": 2}],
"BTC", 'filter_at_the_beginning'), # OperationalException expected
# PrecisionFilter after StaticPairList
([{"method": "StaticPairList"},
{"method": "PrecisionFilter"}],
@ -307,8 +344,8 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
"BTC", 'static_in_the_middle'),
])
def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
pairlists, base_currency, whitelist_result,
caplog) -> None:
ohlcv_history_list, pairlists, base_currency,
whitelist_result, caplog) -> None:
whitelist_conf['pairlists'] = pairlists
whitelist_conf['stake_currency'] = base_currency
@ -324,7 +361,11 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
mocker.patch.multiple('freqtrade.exchange.Exchange',
get_tickers=tickers,
markets=PropertyMock(return_value=shitcoinmarkets),
markets=PropertyMock(return_value=shitcoinmarkets)
)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
)
# Set whitelist_result to None if pairlist is invalid and should produce exception
@ -346,6 +387,10 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
len(whitelist) == whitelist_result
for pairlist in pairlists:
if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
len(ohlcv_history_list) <= pairlist['min_days_listed']:
assert log_has_re(r'^Removed .* from whitelist, because age is less than '
r'.* day.*', caplog)
if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
r'would be <= stop limit.*', caplog)
@ -468,6 +513,29 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_history_list):
mocker.patch.multiple('freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets),
exchange_has=MagicMock(return_value=True),
get_tickers=tickers
)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
)
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_3)
assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
freqtrade.pairlists.refresh_pairlist()
assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count
freqtrade.pairlists.refresh_pairlist()
# Should not have increased since first call.
assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog):
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))