diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a8fc6bc7e..9871ffb89 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -20,9 +20,9 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade -from freqtrade.resolvers import (ExchangeResolver, PairListResolver, - StrategyResolver) +from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets @@ -70,8 +70,7 @@ class FreqtradeBot: # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') - self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist + self.pairlists = PairListManager(self.exchange, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index eb6af9d52..845c5d01f 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -9,25 +9,16 @@ from abc import ABC, abstractmethod from typing import Dict, List from freqtrade.exchange import market_is_active -from freqtrade.pairlist.IPairListFilter import IPairListFilter -from freqtrade.resolvers.pairlistfilter_resolver import PairListFilterResolver logger = logging.getLogger(__name__) class IPairList(ABC): - def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + self._exchange = exchange self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._filters = self._config.get('pairlist', {}).get('filters', {}) - self._pairlistfilters: List[IPairListFilter] = [] - for pl_filter in self._filters.keys(): - self._pairlistfilters.append( - PairListFilterResolver(pl_filter, freqtrade, self._config).pairlistfilter - ) + self._pairlistconfig = pairlistconfig @property def name(self) -> str: @@ -37,22 +28,6 @@ class IPairList(ABC): """ return self.__class__.__name__ - @property - def whitelist(self) -> List[str]: - """ - Has the current whitelist - -> no need to overwrite in subclasses - """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - """ - Has the current blacklist - -> no need to overwrite in subclasses - """ - return self._blacklist - @abstractmethod def short_desc(self) -> str: """ @@ -61,28 +36,16 @@ class IPairList(ABC): """ @abstractmethod - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary -> Please overwrite in subclasses + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ - def validate_whitelist(self, pairlist: List[str], - tickers: List[Dict] = []) -> List[str]: - """ - Validate pairlist against active markets and blacklist. - Run PairlistFilters if these are configured. - """ - pairlist = self._whitelist_for_active_markets(pairlist) - - if not tickers: - # Refresh tickers if they are not used by the parent Pairlist - tickers = self._freqtrade.exchange.get_tickers() - - for pl_filter in self._pairlistfilters: - pairlist = pl_filter.filter_pairlist(pairlist, tickers) - return pairlist - def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary @@ -90,16 +53,14 @@ class IPairList(ABC): :return: the list of pairs the user wants to trade without those unavailable or black_listed """ - markets = self._freqtrade.exchange.markets + markets = self._exchange.markets sanitized_whitelist: List[str] = [] for pair in whitelist: - # pair is not in the generated dynamic market, or in the blacklist ... ignore it - if (pair in self.blacklist or pair not in markets - or not pair.endswith(self._config['stake_currency'])): + # pair is not in the generated dynamic market or has the wrong stake currency + if (pair not in markets or not pair.endswith(self._config['stake_currency'])): logger.warning(f"Pair {pair} is not compatible with exchange " - f"{self._freqtrade.exchange.name} or contained in " - f"your blacklist. Removing it from whitelist..") + f"{self._exchange.name}. Removing it from whitelist..") continue # Check if market is active market = markets[pair] diff --git a/freqtrade/pairlist/IPairListFilter.py b/freqtrade/pairlist/IPairListFilter.py deleted file mode 100644 index 4b43f0e9f..000000000 --- a/freqtrade/pairlist/IPairListFilter.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging -from abc import ABC, abstractmethod -from typing import Dict, List - -logger = logging.getLogger(__name__) - - -class IPairListFilter(ABC): - - def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config - - @abstractmethod - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: - """ - Method doing the filtering - """ diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 778c9b4e0..2f4a3be75 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -2,18 +2,23 @@ import logging from copy import deepcopy from typing import Dict, List -from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class LowPriceFilter(IPairListFilter): +class LowPriceFilter(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + super().__init__(exchange, config, pairlistconfig) - self._low_price_percent = config['pairlist']['filters']['LowPriceFilter'].get( - 'low_price_percent', 0) + self._low_price_percent = pairlistconfig.get('low_price_percent', 0) + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Filtering pairs priced below {self._low_price_percent * 100}%." def _validate_ticker_lowprice(self, ticker) -> bool: """ @@ -22,7 +27,7 @@ class LowPriceFilter(IPairListFilter): :param precision: Precision :return: True if the pair can stay, false if it should be removed """ - precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price'] + precision = self._exchange.markets[ticker['symbol']]['precision']['price'] compare = ticker['last'] + 1 / pow(10, precision) changeperc = (compare - ticker['last']) / ticker['last'] @@ -33,10 +38,14 @@ class LowPriceFilter(IPairListFilter): return True def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: - """ - Method doing the filtering - """ + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ # Copy list since we're modifying this list for p in deepcopy(pairlist): ticker = [t for t in tickers if t['symbol'] == p][0] diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index c720b8e61..0a590bec6 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -2,15 +2,18 @@ import logging from copy import deepcopy from typing import Dict, List -from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class PrecisionFilter(IPairListFilter): +class PrecisionFilter(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Filtering untradable pairs." def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: """ @@ -35,7 +38,7 @@ class PrecisionFilter(IPairListFilter): def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Method doing the filtering + Filters and sorts pairlists and assigns and returns them again. """ if self._freqtrade.strategy.stoploss is not None: # Precalculate sanitized stoploss value to avoid recalculation for every pair diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 074652b25..c10dde5a6 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,6 +5,7 @@ Provides lists as configured in config.json """ import logging +from typing import List, Dict from freqtrade.pairlist.IPairList import IPairList @@ -13,8 +14,8 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + super().__init__(exchange, config, pairlistconfig) def short_desc(self) -> str: """ @@ -23,8 +24,12 @@ class StaticPairList(IPairList): """ return f"{self.name}: {self.whitelist}" - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ - self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) + return self.validate_whitelist(self._config['exchange']['pair_whitelist']) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 911bb3bda..ff601bc44 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import List +from typing import List, Dict from cachetools import TTLCache, cached @@ -19,18 +19,17 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] class VolumePairList(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) - self._whitelistconf = self._config.get('pairlist', {}).get('config') - if 'number_assets' not in self._whitelistconf: + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + super().__init__(exchange, config, pairlistconfig) + + if 'number_assets' not in self._pairlistconfig: raise OperationalException( f'`number_assets` not specified. Please check your configuration ' 'for "pairlist.config.number_assets"') - self._number_pairs = self._whitelistconf['number_assets'] - self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') - self._precision_filter = self._whitelistconf.get('precision_filter', True) + self._number_pairs = self._pairlistconfig['number_assets'] + self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') - if not self._freqtrade.exchange.exchange_has('fetchTickers'): + if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' @@ -45,14 +44,16 @@ class VolumePairList(IPairList): def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages - -> Please overwrite in subclasses """ return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively - -> Please overwrite in subclasses + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ # Generate dynamic whitelist self._whitelist = self._gen_pair_whitelist( @@ -64,17 +65,18 @@ class VolumePairList(IPairList): Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str :param key: sort key (defaults to 'quoteVolume') + :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ - tickers = self._freqtrade.exchange.get_tickers() + tickers = self._exchange.get_tickers() # check length so that we make sure that '/' is actually in the string tickers = [v for k, v in tickers.items() if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency and v[key] is not None)] sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) # Validate whitelist to only have active market pairs - pairs = self.validate_whitelist([s['symbol'] for s in sorted_tickers], tickers) + pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers]) logger.info(f"Searching {self._number_pairs} pairs: {pairs[:self._number_pairs]}") diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py new file mode 100644 index 000000000..9e92fdb6a --- /dev/null +++ b/freqtrade/pairlist/pairlistmanager.py @@ -0,0 +1,68 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from copy import deepcopy +from typing import List + +from freqtrade.pairlist.IPairList import IPairList +from freqtrade.resolvers import PairListResolver + +logger = logging.getLogger(__name__) + + +class PairListManager(): + + def __init__(self, exchange, config: dict) -> None: + self._exchange = exchange + self._config = config + self._whitelist = self._config['exchange'].get('pair_whitelist') + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._pairlists: List[IPairList] = [] + + for pl in self._config.get('pairlists', [{'method': "StaticPairList"}]): + pairl = PairListResolver(pl.get('method'), + exchange, config, + pl.get('config')).pairlist + self._pairlists.append(pairl) + + @property + def whitelist(self) -> List[str]: + """ + Has the current whitelist + """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ + return self._blacklist + + def refresh_pairlist(self) -> None: + """ + Run pairlist through all pairlists. + """ + + pairlist = self._whitelist.copy() + + # tickers should be cached to avoid calling the exchange on each call. + tickers = self._exchange.get_tickers() + for pl in self._pairlists: + pl.filter_pairlist(pairlist, tickers) + + pairlist = self._verify_blacklist(pairlist) + self._whitelist = pairlist + + def _verify_blacklist(self, pairlist: List[str]) -> List[str]: + + for pair in deepcopy(pairlist): + if pair in self.blacklist: + logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") + pairlist.remove(pair) + return pairlist diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 0f23bb3fd..db00f6515 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -20,13 +20,15 @@ class PairListResolver(IResolver): __slots__ = ['pairlist'] - def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + def __init__(self, pairlist_name: str, exchange, config: dict, pairlistconfig) -> None: """ Load the custom class from config parameter :param config: configuration dictionary or None """ - self.pairlist = self._load_pairlist(pairlist_name, config, kwargs={'freqtrade': freqtrade, - 'config': config}) + self.pairlist = self._load_pairlist(pairlist_name, config, + kwargs={'exchange': exchange, + 'config': config, + 'pairlistconfig': pairlistconfig}) def _load_pairlist( self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList: diff --git a/freqtrade/resolvers/pairlistfilter_resolver.py b/freqtrade/resolvers/pairlistfilter_resolver.py deleted file mode 100644 index bf86d1c6c..000000000 --- a/freqtrade/resolvers/pairlistfilter_resolver.py +++ /dev/null @@ -1,53 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom pairlists -""" -import logging -from pathlib import Path - -from freqtrade import OperationalException -from freqtrade.pairlist.IPairListFilter import IPairListFilter -from freqtrade.resolvers import IResolver - -logger = logging.getLogger(__name__) - - -class PairListFilterResolver(IResolver): - """ - This class contains all the logic to load custom PairList class - """ - - __slots__ = ['pairlistfilter'] - - def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: - """ - Load the custom class from config parameter - :param config: configuration dictionary or None - """ - self.pairlistfilter = self._load_pairlist(pairlist_name, config, - kwargs={'freqtrade': freqtrade, - 'config': config}) - - def _load_pairlist( - self, pairlistfilter_name: str, config: dict, kwargs: dict) -> IPairListFilter: - """ - Search and loads the specified pairlist. - :param pairlistfilter_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given pairlist - :return: PairList instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() - - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=None, extra_dir=None) - - pairlist = self._load_object(paths=abs_paths, object_type=IPairListFilter, - object_name=pairlistfilter_name, kwargs=kwargs) - if pairlist: - return pairlist - raise OperationalException( - f"Impossible to load PairlistFilter '{pairlistfilter_name}'. This class does not exist " - "or contains Python code errors." - )