From 3e2fa580291066b3c4fd0948a4fdf0eea6935569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 20:44:56 +0100 Subject: [PATCH] load pairlists via resolver --- docs/developer.md | 14 +++- freqtrade/freqtradebot.py | 12 ++-- freqtrade/pairlist/IPairList.py | 91 ++++++++++++++++++++++++ freqtrade/pairlist/StaticPairList.py | 72 ++----------------- freqtrade/pairlist/VolumePairList.py | 16 ++--- freqtrade/resolvers/__init__.py | 1 + freqtrade/resolvers/pairlist_resolver.py | 59 +++++++++++++++ 7 files changed, 179 insertions(+), 86 deletions(-) create mode 100644 freqtrade/pairlist/IPairList.py create mode 100644 freqtrade/resolvers/pairlist_resolver.py diff --git a/docs/developer.md b/docs/developer.md index 35410de1f..c564acffb 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -20,6 +20,16 @@ This is a simple provider, which however serves as a good example on how to star Next, modify the classname of the provider (ideally align this with the Filename). +The base-class provides the an instance of the bot (`self._freqtrade`), as well as the configuration (`self._config`), and initiates both `_blacklist` and `_whitelist`. + +```python + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) +``` + + Now, let's step through the methods which require actions: #### configuration @@ -35,7 +45,7 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke Returns a description used for Telegram messages. This should coutain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. -#### refresh_whitelist +#### refresh_pairlist Override this method and run all calculations needed in this method. This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. @@ -47,7 +57,7 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs ##### sample ``` python - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: # Generate dynamic whitelist pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) # Validate whitelist to only have active market pairs diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 54374b209..01de328f6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,12 +19,10 @@ from freqtrade.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers import StrategyResolver, PairListResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -from freqtrade.pairlist.StaticPairList import StaticPairList -from freqtrade.pairlist.VolumePairList import VolumePairList logger = logging.getLogger(__name__) @@ -59,10 +57,8 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - if self.config.get('pairlist', {}).get('method') == 'VolumePairList': - self.pairlists: StaticPairList = VolumePairList(self, self.config) - else: - self.pairlists: StaticPairList = StaticPairList(self, self.config) + pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') + self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -151,7 +147,7 @@ class FreqtradeBot(object): state_changed = False try: # Refresh whitelist - self.pairlists.refresh_whitelist() + self.pairlists.refresh_pairlist() self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py new file mode 100644 index 000000000..6b5b0db4b --- /dev/null +++ b/freqtrade/pairlist/IPairList.py @@ -0,0 +1,91 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from abc import ABC, abstractmethod +from typing import List + +logger = logging.getLogger(__name__) + + +class IPairList(ABC): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + + @property + def name(self) -> str: + """ + Gets name of the class + -> no need to overwrite in subclasses + """ + return self.__class__.__name__ + + @property + def whitelist(self) -> List[str]: + """ + Has the current whitelist + -> no need to overwrite in subclasses + """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ + return self._blacklist + + @abstractmethod + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + + @abstractmethod + def refresh_pairlist(self) -> None: + """ + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + -> Please overwrite in subclasses + """ + + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 5b0e37357..5896e814a 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,43 +5,16 @@ Provides lists as configured in config.json """ import logging -from typing import List + +from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class StaticPairList(object): +class StaticPairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) - # self.refresh_whitelist() - - @property - def name(self) -> str: - """ - Gets name of the class - -> no need to overwrite in subclasses - """ - return self.__class__.__name__ - - @property - def whitelist(self) -> List[str]: - """ - Has the current whitelist - -> no need to overwrite in subclasses - """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - """ - Has the current blacklist - -> no need to overwrite in subclasses - """ - return self._blacklist + super().__init__(freqtrade, config) def short_desc(self) -> str: """ @@ -50,41 +23,8 @@ class StaticPairList(object): """ return f"{self.name}: {self.whitelist}" - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: """ - Refreshes whitelist and assigns it to self._whitelist + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively """ self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) - - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: - """ - Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or - black_listed - """ - sanitized_whitelist = whitelist - markets = self._freqtrade.exchange.get_markets() - - # Filter to markets in stake currency - markets = [m for m in markets if m['quote'] == self._config['stake_currency']] - known_pairs = set() - - for market in markets: - pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.blacklist: - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not market['active']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair - ) - - # We need to remove pairs that are unknown - return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index bd562da83..9d3d46169 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -8,21 +8,18 @@ import logging from typing import List from cachetools import TTLCache, cached -from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade.pairlist.IPairList import IPairList from freqtrade import OperationalException logger = logging.getLogger(__name__) SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] -class VolumePairList(StaticPairList): +class VolumePairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config + super().__init__(freqtrade, config) self._whitelistconf = self._config.get('pairlist', {}).get('config') - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') @@ -34,7 +31,6 @@ class VolumePairList(StaticPairList): if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') - # self.refresh_whitelist() def _validate_keys(self, key): return key in SORT_VALUES @@ -46,9 +42,10 @@ class VolumePairList(StaticPairList): """ return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: """ - Refreshes whitelist and assigns it to self._whitelist + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + -> Please overwrite in subclasses """ # Generate dynamic whitelist pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) @@ -72,4 +69,3 @@ class VolumePairList(StaticPairList): sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) pairs = [s['symbol'] for s in sorted_tickers] return pairs - diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index 84e3bcdcd..da2987b27 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -1,3 +1,4 @@ from freqtrade.resolvers.iresolver import IResolver # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 +from freqtrade.resolvers.pairlist_resolver import PairListResolver # noqa: F401 from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401 diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py new file mode 100644 index 000000000..286cea5bf --- /dev/null +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -0,0 +1,59 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import logging +from pathlib import Path + +from freqtrade.pairlist.IPairList import IPairList +from freqtrade.resolvers import IResolver + +logger = logging.getLogger(__name__) + + +class PairListResolver(IResolver): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['pairlist'] + + def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + self.pairlist = self._load_pairlist(pairlist_name, kwargs={'freqtrade': freqtrade, + 'config': config}) + + def _load_pairlist( + self, pairlist_name: str, kwargs: dict) -> IPairList: + """ + Search and loads the specified pairlist. + :param pairlist_name: name of the module to import + :param extra_dir: additional directory to search for the given pairlist + :return: PairList instance or None + """ + current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() + + abs_paths = [ + current_path.parent.parent.joinpath('user_data/pairlist'), + current_path, + ] + + for _path in abs_paths: + try: + pairlist = self._search_object(directory=_path, object_type=IPairList, + object_name=pairlist_name, + kwargs=kwargs) + if pairlist: + logger.info('Using resolved pairlist %s from \'%s\'', pairlist_name, _path) + return pairlist + except FileNotFoundError: + logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) + + raise ImportError( + "Impossible to load Pairlist '{}'. This class does not exist" + " or contains Python code errors".format(pairlist_name) + )