diff --git a/freqtrade/pairlist/AgeFilter.py b/freqtrade/pairlist/AgeFilter.py index f909014ba..21e1b1a01 100644 --- a/freqtrade/pairlist/AgeFilter.py +++ b/freqtrade/pairlist/AgeFilter.py @@ -2,7 +2,7 @@ Minimum age (days listed) pair list filter """ import logging -from typing import Any, Dict +from typing import Any, Dict, List import arrow @@ -49,36 +49,32 @@ class AgeFilter(IPairList): return (f"{self.name} - Filtering pairs with age less than " f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") - def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ - Validate age for the ticker - :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, false if it should be removed + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist """ - - # Check symbol in cache - if ticker['symbol'] in self._symbolsChecked: - return True + needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] + if not needed_pairs: + return pairlist since_ms = int(arrow.utcnow() .floor('day') - .shift(days=-self._min_days_listed) + .shift(days=-self._min_days_listed - 1) .float_timestamp) * 1000 + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) + pairlist_new = [] + if self._enabled: + for p, _ in needed_pairs: - daily_candles = self._exchange.get_historic_ohlcv(pair=ticker['symbol'], - timeframe='1d', - since_ms=since_ms) - - if daily_candles is not None: - if len(daily_candles) > self._min_days_listed: - # We have fetched at least the minimum required number of daily candles - # Add to cache, store the time we last checked this symbol - self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000 - return True - else: - self.log_once(f"Removed {ticker['symbol']} from whitelist, because age " - f"{len(daily_candles)} is less than {self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')}", logger.info) - return False - return False + age = len(candles[(p, '1d')]) if (p, '1d') in candles else 0 + if age > self._min_days_listed: + pairlist_new.append(p) + self._symbolsChecked[p] = int(arrow.utcnow().float_timestamp) * 1000 + else: + self.log_once(f"Removed {p} from whitelist, because age " + f"{age} is less than {self._min_days_listed} " + f"{plural(self._min_days_listed, 'day')}", logger.info) + logger.info(f"Validated {len(pairlist_new)} pairs.") + return pairlist_new diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 810a22300..418cc9e92 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -3,7 +3,7 @@ PairList manager class """ import logging from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from cachetools import TTLCache, cached @@ -97,7 +97,7 @@ class PairListManager(): self._whitelist = pairlist - def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]: + def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]: """ Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove pairs that do not have ticker available diff --git a/freqtrade/pairlist/rangestabilityfilter.py b/freqtrade/pairlist/rangestabilityfilter.py index f1fecc59c..8efded9ee 100644 --- a/freqtrade/pairlist/rangestabilityfilter.py +++ b/freqtrade/pairlist/rangestabilityfilter.py @@ -1,8 +1,9 @@ """ Rate of change pairlist filter """ +from copy import deepcopy import logging -from typing import Any, Dict +from typing import Any, Dict, List import arrow from cachetools.ttl import TTLCache @@ -51,7 +52,33 @@ class RangeStabilityFilter(IPairList): return (f"{self.name} - Filtering pairs with rate of change below " f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.") - def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Validate trading range + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .float_timestamp) * 1000 + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, + cache=False) + + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + return pairlist + + def _validate_pair_loc(self, pair: str, daily_candles: Dict[str, Any]) -> bool: """ Validate trading range :param pair: Pair that's currently validated @@ -62,14 +89,6 @@ class RangeStabilityFilter(IPairList): if pair in self._pair_cache: return self._pair_cache[pair] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days) - .float_timestamp) * 1000 - - daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair, - timeframe='1d', - since_ms=since_ms) result = False if daily_candles is not None and not daily_candles.empty: highest_high = daily_candles['high'].max() @@ -79,7 +98,7 @@ class RangeStabilityFilter(IPairList): result = True else: self.log_once(f"Removed {pair} from whitelist, because rate of change " - f"over {plural(self._days, 'day')} is {pct_change:.3f}, " + f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " f"which is below the threshold of {self._min_rate_of_change}.", logger.info) result = False