From afaca2167cce2531817e9f65bb0dd0343d331083 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Oct 2022 19:33:02 +0000 Subject: [PATCH] use Type Alias for Ticker result to improve keyerror resiliancy --- freqtrade/exchange/exchange.py | 6 ++++-- freqtrade/exchange/kraken.py | 3 ++- freqtrade/exchange/types.py | 5 +++++ freqtrade/plugins/pairlist/AgeFilter.py | 3 ++- freqtrade/plugins/pairlist/IPairList.py | 11 ++++++----- freqtrade/plugins/pairlist/OffsetFilter.py | 3 ++- freqtrade/plugins/pairlist/PerformanceFilter.py | 3 ++- freqtrade/plugins/pairlist/PrecisionFilter.py | 7 ++++--- freqtrade/plugins/pairlist/PriceFilter.py | 7 ++++--- freqtrade/plugins/pairlist/ProducerPairList.py | 5 +++-- freqtrade/plugins/pairlist/ShuffleFilter.py | 3 ++- freqtrade/plugins/pairlist/SpreadFilter.py | 7 ++++--- freqtrade/plugins/pairlist/StaticPairList.py | 5 +++-- freqtrade/plugins/pairlist/VolatilityFilter.py | 3 ++- freqtrade/plugins/pairlist/VolumePairList.py | 7 ++++--- freqtrade/plugins/pairlist/rangestabilityfilter.py | 3 ++- freqtrade/plugins/pairlistmanager.py | 3 ++- 17 files changed, 53 insertions(+), 31 deletions(-) create mode 100644 freqtrade/exchange/types.py diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c41a84450..3b59bdfa0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -31,6 +31,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, remove_credentials, retrier, retrier_async) +from freqtrade.exchange.types import Tickers from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json, safe_value_fallback2) from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -1420,14 +1421,15 @@ class Exchange: raise OperationalException(e) from e @retrier - def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict: + def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Tickers: """ :param cached: Allow cached result :return: fetch_tickers result """ + tickers: Tickers if cached: with self._cache_lock: - tickers = self._fetch_tickers_cache.get('fetch_tickers') + tickers = self._fetch_tickers_cache.get('fetch_tickers') # type: ignore if tickers: return tickers try: diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 6c9b88eae..f3a9486f2 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -12,6 +12,7 @@ from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, Invali OperationalException, TemporaryError) from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier +from freqtrade.exchange.types import Tickers logger = logging.getLogger(__name__) @@ -45,7 +46,7 @@ class Kraken(Exchange): return (parent_check and market.get('darkpool', False) is False) - def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict: + def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Tickers: # Only fetch tickers for current stake currency # Otherwise the request for kraken becomes too large. symbols = list(self.get_markets(quote_currencies=[self._config['stake_currency']])) diff --git a/freqtrade/exchange/types.py b/freqtrade/exchange/types.py new file mode 100644 index 000000000..6504381dc --- /dev/null +++ b/freqtrade/exchange/types.py @@ -0,0 +1,5 @@ +from typing import Any, Dict + + +Ticker = Dict[str, Any] +Tickers = Dict[str, Ticker] diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 7c8cdb5ab..f9c02e250 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -10,6 +10,7 @@ from pandas import DataFrame from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.util import PeriodicCache @@ -67,7 +68,7 @@ class AgeFilter(IPairList): f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else '') - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ :param pairlist: pairlist to filter or sort :param tickers: Tickers (from exchange.get_tickers). May be cached. diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 60abac6a1..660d6228c 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -4,11 +4,12 @@ PairList Handler base class import logging from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange, market_is_active +from freqtrade.exchange.types import Ticker, Tickers from freqtrade.mixins import LoggingMixin @@ -61,7 +62,7 @@ class IPairList(LoggingMixin, ABC): -> Please overwrite in subclasses """ - def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check one pair against Pairlist Handler's specific conditions. @@ -74,7 +75,7 @@ class IPairList(LoggingMixin, ABC): """ raise NotImplementedError() - def gen_pairlist(self, tickers: Dict) -> List[str]: + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist. @@ -91,7 +92,7 @@ class IPairList(LoggingMixin, ABC): raise OperationalException("This Pairlist Handler should not be used " "at the first position in the list of Pairlist Handlers.") - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. @@ -110,7 +111,7 @@ class IPairList(LoggingMixin, ABC): # Copy list since we're modifying this list for p in deepcopy(pairlist): # Filter out assets - if not self._validate_pair(p, tickers[p] if p in tickers else {}): + if not self._validate_pair(p, tickers[p] if p in tickers else None): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index c9531ece1..8f21cdd85 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList @@ -42,7 +43,7 @@ class OffsetFilter(IPairList): return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." return f"{self.name} - Offsetting pairs by {self._offset}." - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 4cc92175a..e7fcac1e4 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List import pandas as pd from freqtrade.constants import Config +from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.IPairList import IPairList @@ -39,7 +40,7 @@ class PerformanceFilter(IPairList): """ return f"{self.name} - Sorting pairs by performance." - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the allowlist again. Called on each bot iteration - please use internal caching if necessary diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 98cb3ba46..478eaec20 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -2,10 +2,11 @@ Precision pair list filter """ import logging -from typing import Any, Dict +from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Ticker from freqtrade.plugins.pairlist.IPairList import IPairList @@ -44,7 +45,7 @@ class PrecisionFilter(IPairList): """ return f"{self.name} - Filtering untradable pairs." - def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very low value pairs. @@ -52,7 +53,7 @@ class PrecisionFilter(IPairList): :param ticker: ticker dict as returned from ccxt.fetch_ticker :return: True if the pair can stay, false if it should be removed """ - if ticker.get('last', None) is None: + if not ticker or ticker.get('last', None) is None: self.log_once(f"Removed {pair} from whitelist, because " "ticker['last'] is empty (Usually no trade in the last 24h).", logger.info) diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index a6b400a38..f007c52d2 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -2,10 +2,11 @@ Price pair list filter """ import logging -from typing import Any, Dict +from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Ticker from freqtrade.plugins.pairlist.IPairList import IPairList @@ -64,14 +65,14 @@ class PriceFilter(IPairList): return f"{self.name} - No price filters configured." - def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if if one price-step (pip) is > than a certain barrier. :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.fetch_ticker :return: True if the pair can stay, false if it should be removed """ - if ticker.get('last', None) is None or ticker.get('last') == 0: + if not ticker or ticker.get('last', None) is None or ticker.get('last') == 0: self.log_once(f"Removed {pair} from whitelist, because " "ticker['last'] is empty (Usually no trade in the last 24h).", logger.info) diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 740cb4ec2..882d49b76 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -7,6 +7,7 @@ import logging from typing import Any, Dict, List, Optional from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList @@ -68,7 +69,7 @@ class ProducerPairList(IPairList): return pairs - def gen_pairlist(self, tickers: Dict) -> List[str]: + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist :param tickers: Tickers (from exchange.get_tickers). May be cached. @@ -79,7 +80,7 @@ class ProducerPairList(IPairList): pairs = self._whitelist_for_active_markets(self.verify_whitelist(pairs, logger.info)) return pairs - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 6eb4231bc..1bc114d4e 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.enums import RunMode +from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList @@ -47,7 +48,7 @@ class ShuffleFilter(IPairList): return (f"{self.name} - Shuffling pairs" + (f", seed = {self._seed}." if self._seed is not None else ".")) - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 1f6d4f687..ecb21593b 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -2,10 +2,11 @@ Spread pair list filter """ import logging -from typing import Any, Dict +from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Ticker from freqtrade.plugins.pairlist.IPairList import IPairList @@ -44,14 +45,14 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio:.2%}.") - def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Validate spread for the ticker :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.fetch_ticker :return: True if the pair can stay, false if it should be removed """ - if 'bid' in ticker and 'ask' in ticker and ticker['ask'] and ticker['bid']: + if ticker and 'bid' in ticker and 'ask' in ticker and ticker['ask'] and ticker['bid']: spread = 1 - ticker['bid'] / ticker['ask'] if spread > self._max_spread_ratio: self.log_once(f"Removed {pair} from whitelist, because spread " diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 5b1337754..4b1961a53 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any, Dict, List from freqtrade.constants import Config +from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList @@ -39,7 +40,7 @@ class StaticPairList(IPairList): """ return f"{self.name}" - def gen_pairlist(self, tickers: Dict) -> List[str]: + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist :param tickers: Tickers (from exchange.get_tickers). May be cached. @@ -53,7 +54,7 @@ class StaticPairList(IPairList): return self._whitelist_for_active_markets( self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)) - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index c06fc09ba..401a2e86c 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -13,6 +13,7 @@ from pandas import DataFrame from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -62,7 +63,7 @@ class VolatilityFilter(IPairList): f"{self._min_volatility}-{self._max_volatility} " f" the last {self._days} {plural(self._days, 'day')}.") - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range :param pairlist: pairlist to filter or sort diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index bfecbd62a..ad27a93d8 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -5,13 +5,14 @@ Provides dynamic pair list based on trade volumes """ import logging from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List +from typing import Any, Dict, List, Literal from cachetools import TTLCache from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date +from freqtrade.exchange.types import Tickers from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.IPairList import IPairList @@ -36,7 +37,7 @@ class VolumePairList(IPairList): self._stake_currency = config['stake_currency'] self._number_pairs = self._pairlistconfig['number_assets'] - self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') + self._sort_key: Literal['quoteVolume'] = self._pairlistconfig.get('sort_key', 'quoteVolume') self._min_value = self._pairlistconfig.get('min_value', 0) self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) @@ -110,7 +111,7 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." - def gen_pairlist(self, tickers: Dict) -> List[str]: + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist :param tickers: Tickers (from exchange.get_tickers). May be cached. diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index ca844f003..546b026cb 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -11,6 +11,7 @@ from pandas import DataFrame from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -60,7 +61,7 @@ class RangeStabilityFilter(IPairList): f"{self._min_rate_of_change}{max_rate_desc} over the " f"last {plural(self._days, 'day')}.") - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range :param pairlist: pairlist to filter or sort diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 5ed319e93..1e98b8485 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -11,6 +11,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers from freqtrade.mixins import LoggingMixin from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -76,7 +77,7 @@ class PairListManager(LoggingMixin): return [{p.name: p.short_desc()} for p in self._pairlist_handlers] @cached(TTLCache(maxsize=1, ttl=1800)) - def _get_cached_tickers(self): + def _get_cached_tickers(self) -> Tickers: return self._exchange.get_tickers() def refresh_pairlist(self) -> None: