From b65a04dd908682fd83d5e780c43c2a65a8bab85d Mon Sep 17 00:00:00 2001 From: Bernd Zeimetz Date: Tue, 1 Jun 2021 21:23:57 +0200 Subject: [PATCH] Add MarketCapPairList. After recurring requests on #strategy-chat, this implements a pairlist based on market capitalization. Data is retrieved from the coingecko API https://www.coingecko.com/en/api --- docs/includes/pairlists.md | 26 ++ freqtrade/constants.py | 2 +- freqtrade/optimize/backtesting.py | 2 + .../plugins/pairlist/MarketCapPairList.py | 263 ++++++++++++++++++ 4 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 freqtrade/plugins/pairlist/MarketCapPairList.py diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index ce0cc6e57..ce51a893d 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -22,6 +22,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) +* [`MarketCapPairList`](#market-capitalization-pair-list) * [`AgeFilter`](#agefilter) * [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) @@ -79,6 +80,31 @@ Filtering instances (not the first position in the list) will not apply any cach !!! Note `VolumePairList` does not support backtesting mode. +#### Market Capitalization Pair List + +`MarketCapPairList` employs sorting/filtering of pairs by their market capitalization. It selects `number_assets` top pairs + +When used in the chain of Pairlist Handlers in a non-leading position (after StaticPairList and other Pairlist Filters), `MarketCapPairList` considers outputs of previous Pairlist Handlers, adding its sorting/selection of the pairs by the market capitalization. + +When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top pairs based on market capitalization with matching stake-currency on the exchange. + +The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 21600s (6 hours). +The pairlist cache (`refresh_period`) on `MarketCapPairList` is only applicable to generating pairlists. +Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data. + +`MarketCapPairList` is based on the data from `coingecko.com`. + +```json +"pairlists": [{ + "method": "MarketCapPairList", + "number_assets": 20, + "refresh_period": 1800 +}], +``` + +!!! Note + `MarketCapPairList` does not support backtesting mode. + #### AgeFilter Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`). diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e42b9d4b8..eef2f1fdb 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,7 +24,7 @@ ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'MarketCapPairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cbc0995aa..af3d34ced 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -84,6 +84,8 @@ class Backtesting: self.timeframe_min = timeframe_to_minutes(self.timeframe) self.pairlists = PairListManager(self.exchange, self.config) + if 'MarketCapPairList' in self.pairlists.name_list: + raise OperationalException("MarketCapPairList not allowed for backtesting.") if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting.") if 'PerformanceFilter' in self.pairlists.name_list: diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py new file mode 100644 index 000000000..3fb3d4131 --- /dev/null +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -0,0 +1,263 @@ +""" +MarketCap PairList provider + +Provides dynamic pair list based on market cap +""" +import logging +from typing import Any, Dict, List + +from cachetools.ttl import TTLCache +from pandas.core.common import flatten +from pycoingecko import CoinGeckoAPI + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import chunks +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class MarketCapPairList(IPairList): + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if 'number_assets' not in self._pairlistconfig: + raise OperationalException( + '`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') + if (not type(self._pairlistconfig['number_assets']) is int) or ( + int(self._pairlistconfig['number_assets']) <= 0): + raise OperationalException( + '"number_assets" should be a positive integer. ' + 'Please edit your config and restart the bot.' + ) + self._number_pairs = int(self._pairlistconfig['number_assets']) + + self._stake_currency = config['stake_currency'] + self._refresh_period = self._pairlistconfig.get('refresh_period', 6*60*60) + self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._cg = CoinGeckoAPI() + self._marketcap_ranks: Dict[str, Any] = {} + self._coins_list: Dict[str, str] = {} + + if not self._exchange.exchange_has('fetchTickers'): + raise OperationalException( + 'Exchange does not support dynamic whitelist. ' + 'Please edit your config and restart the bot.' + ) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - top {self._pairlistconfig['number_assets']} marketcap pairs." + + def _request_coins_list(self): + """ + Update the symbol -> coin id mapping. Coin ids are needed to request market data + for single coins. + """ + coins_list = self._cg.get_coins_list() + for x in coins_list: + # yes, lower() is needed as there are a few uppercase symbols + symbol = x['symbol'].lower() + cid = x['id'] + + # ignore binance-peg- + if 'binance-peg-' in cid: + continue + # ignore -bsc + if cid.endswith('-bsc'): + continue + # yes, not all symbols are unique. + if symbol in self._coins_list: + # in a few cases the id matches the symbol + # we have that version already, keep it. + if self._coins_list[symbol] == symbol: + continue + # the new one matches, use it. + elif cid == symbol: + self._coins_list[symbol] + else: + self._coins_list[symbol] = list(flatten([self._coins_list[symbol], cid])) + else: + self._coins_list[symbol] = cid + + def _request_marketcap_ranks(self): + """ + To keep the request count low, download the most common coins. + Then update all coins that where not in the top 250 ranks. + """ + ids = [] + + # there might be some stuff to add here.. + # probably also needs to be configurable and/or have + # exchange specific settings + symbol_map = { + 'acm': 'ac-milan-fan-token', + 'ant': 'aragon', + 'atm': 'atletico-madrid', + 'bat': 'basic-attention-token', + 'bts': 'bitshares', + 'comp': 'compound-governance-token', + 'dego': 'dego-finance', + 'eps': 'ellipsis', + 'grt': 'the-graph', + 'hnt': 'helium', + 'hot': 'holotoken', + 'iota': 'miota', + 'lit': 'litentry', + 'luna': 'terra-luna', + 'mask': 'mask-network', + 'mdx': 'mdex', + 'mir': 'mirror-protocol', + 'og': 'og-fan-token', + 'one': 'harmony', + 'pax': 'paxos-standard', + 'pnt': 'pnetwork', + 'rune': 'thorchain', + 'sand': 'the-sandbox', + 'stx': 'blockstack', + 'super': 'superfarm', + 'tct': 'tokenclub', + 'trb': 'tellor', + 'tru': 'truefi', + 'trx': 'tron', + 'uni': 'uniswap', + 'wing': 'wing-finance', + } + + for symbol in self._marketcap_ranks.keys(): + # symbol needs mapping + if symbol in symbol_map: + ids.append(symbol_map[symbol]) + elif symbol in self._coins_list: + _id = self._coins_list[symbol] + # symbol is not unique (sigh!) + if type(_id) == list: + pair = symbol.upper() + f'/{self._stake_currency}' + _id_text = ', '.join(_id) + self.log_once(f'Symbol for {pair} is not unique on coingecko ' + f'({_id_text}), dropping.', logger.warning) + continue + ids.append(_id) + elif symbol in self._coins_list.values(): + # try to add some symbols automatically + symbol_map[symbol] = [k for k, v in self._coins_list.items() if v == symbol][0] + ids.append(symbol) + + # reverse map + rev_symbol_map = {v: k for k, v in symbol_map.items()} + + _marketcap_ranks = {} + + # seems coingecko limits the number of ids to 52, using 50. + for _ids in chunks(ids, 50): + base_marketkaps = self._cg.get_coins_markets( + vs_currency='usd', + ids=','.join(_ids), + order='market_cap_desc', + per_page=len(ids), + sparkline=False, + page=1) + for x in base_marketkaps: + # same here, keep symbol lowercase. + _symbol = x['symbol'].lower() + if _symbol in rev_symbol_map: + _symbol = rev_symbol_map[_symbol] + _marketcap_ranks[_symbol] = x['market_cap_rank'] + + for x in self._marketcap_ranks.keys(): + if x not in _marketcap_ranks: + _marketcap_ranks[x] = None + self._marketcap_ranks = _marketcap_ranks + + def update_marketcap_ranks(self, symbols: List[str]): + """ + Updates the dict containing the marketcap ranks of the requested + list of symbols, if needed. + """ + marketcaps_uptodate = self._marketcap_cache.get('marketcaps_uptodate') + startup = not (self._marketcap_ranks and self._coins_list) + + for symbol in symbols: + if symbol.lower() not in self._marketcap_ranks: + marketcaps_uptodate = False + self._marketcap_ranks[symbol.lower()] = None + if not marketcaps_uptodate: + try: + self._request_coins_list() + self._request_marketcap_ranks() + except Exception as e: + if startup: + raise OperationalException( + f'Failed to download marketcap data from coingecko: {e}' + ) + else: + self.log_once( + 'Failed to update marketcap data from coingecko: .' + f'{e}. Using old data.', + logger.warning) + else: + self._marketcap_cache['marketcaps_uptodate'] = True + + def gen_pairlist(self, tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: List of pairs + """ + + # using the pairlist from the exchange works, but if the list is long, it will create a very + # long startup time due to coingecko rate limits. Might be worth to add an config option + # with a big warning. + pairlist = self._whitelist_for_active_markets( + self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)) + return self.filter_pairlist(pairlist, tickers) + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + symbols = [p.replace(f'/{self._stake_currency}', '').lower() for p in pairlist] + self.update_marketcap_ranks(symbols) + + marketcap_ranks = {} + for symbol in symbols: + rank = self._marketcap_ranks[symbol] + if not rank: + pair = symbol.upper() + f'/{self._stake_currency}' + self.log_once(f'No known marketcap for {pair}, dropping.', logger.warning) + continue + marketcap_ranks[symbol] = rank + + # sort marketcaps + pairlist = [ + k.upper() + f'/{self._stake_currency}' + for k, v in sorted(marketcap_ranks.items(), key=lambda x: x[1]) + ] + + # Validate whitelist to only have active market pairs + pairlist = self._whitelist_for_active_markets(pairlist) + pairlist = self.verify_blacklist(pairlist, logger.info) + # Limit pairlist to the requested number of pairs + pairlist = pairlist[:self._number_pairs] + self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info) + + return pairlist