Use async to get candle data for pairlists

freqtrade/pairlist/AgeFilter.py

@@ -2,7 +2,7 @@
 Minimum age (days listed) pair list filter
 """
 import logging
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import arrow
 
@@ -49,36 +49,32 @@ class AgeFilter(IPairList):
         return (f"{self.name} - Filtering pairs with age less than "
                 f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
 
-    def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
-        Validate age for the ticker
+        :param pairlist: pairlist to filter or sort
-        :param pair: Pair that's currently validated
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :return: new allowlist
-        :return: True if the pair can stay, false if it should be removed
         """
-
+        needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
-        # Check symbol in cache
+        if not needed_pairs:
-        if ticker['symbol'] in self._symbolsChecked:
+            return pairlist
-            return True
 
         since_ms = int(arrow.utcnow()
                        .floor('day')
-                       .shift(days=-self._min_days_listed)
+                       .shift(days=-self._min_days_listed - 1)
                        .float_timestamp) * 1000
+        candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
+        pairlist_new = []
+        if self._enabled:
+            for p, _ in needed_pairs:
 
-        daily_candles = self._exchange.get_historic_ohlcv(pair=ticker['symbol'],
+                age = len(candles[(p, '1d')]) if (p, '1d') in candles else 0
-                                                          timeframe='1d',
+                if age > self._min_days_listed:
-                                                          since_ms=since_ms)
+                    pairlist_new.append(p)
-
+                    self._symbolsChecked[p] = int(arrow.utcnow().float_timestamp) * 1000
-        if daily_candles is not None:
+                else:
-            if len(daily_candles) > self._min_days_listed:
+                    self.log_once(f"Removed {p} from whitelist, because age "
-                # We have fetched at least the minimum required number of daily candles
+                                  f"{age} is less than {self._min_days_listed} "
-                # Add to cache, store the time we last checked this symbol
+                                  f"{plural(self._min_days_listed, 'day')}", logger.info)
-                self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000
+        logger.info(f"Validated {len(pairlist_new)} pairs.")
-                return True
+        return pairlist_new
-            else:
-                self.log_once(f"Removed {ticker['symbol']} from whitelist, because age "
-                              f"{len(daily_candles)} is less than {self._min_days_listed} "
-                              f"{plural(self._min_days_listed, 'day')}", logger.info)
-                return False
-        return False

freqtrade/pairlist/pairlistmanager.py

@@ -3,7 +3,7 @@ PairList manager class
 """
 import logging
 from copy import deepcopy
-from typing import Dict, List
+from typing import Any, Dict, List
 
 from cachetools import TTLCache, cached
 
@@ -97,7 +97,7 @@ class PairListManager():
 
         self._whitelist = pairlist
 
-    def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]:
+    def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
         """
         Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
         pairs that do not have ticker available

freqtrade/pairlist/rangestabilityfilter.py

@@ -1,8 +1,9 @@
 """
 Rate of change pairlist filter
 """
+from copy import deepcopy
 import logging
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import arrow
 from cachetools.ttl import TTLCache
@@ -51,7 +52,33 @@ class RangeStabilityFilter(IPairList):
         return (f"{self.name} - Filtering pairs with rate of change below "
                 f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
 
-    def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Validate trading range
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new allowlist
+        """
+        needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
+
+        since_ms = int(arrow.utcnow()
+                       .floor('day')
+                       .shift(days=-self._days - 1)
+                       .float_timestamp) * 1000
+        # Get all candles
+        candles = {}
+        if needed_pairs:
+            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
+                                                          cache=False)
+
+        if self._enabled:
+            for p in deepcopy(pairlist):
+                daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
+                if not self._validate_pair_loc(p, daily_candles):
+                    pairlist.remove(p)
+        return pairlist
+
+    def _validate_pair_loc(self, pair: str, daily_candles: Dict[str, Any]) -> bool:
         """
         Validate trading range
         :param pair: Pair that's currently validated
@@ -62,14 +89,6 @@ class RangeStabilityFilter(IPairList):
         if pair in self._pair_cache:
             return self._pair_cache[pair]
 
-        since_ms = int(arrow.utcnow()
-                       .floor('day')
-                       .shift(days=-self._days)
-                       .float_timestamp) * 1000
-
-        daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair,
-                                                                timeframe='1d',
-                                                                since_ms=since_ms)
         result = False
         if daily_candles is not None and not daily_candles.empty:
             highest_high = daily_candles['high'].max()
@@ -79,7 +98,7 @@ class RangeStabilityFilter(IPairList):
                 result = True
             else:
                 self.log_once(f"Removed {pair} from whitelist, because rate of change "
-                              f"over {plural(self._days, 'day')} is {pct_change:.3f}, "
+                              f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
                               f"which is below the threshold of {self._min_rate_of_change}.",
                               logger.info)
                 result = False