Merge pull request #1596 from iuvbio/feature/volume-precision-pairlist

Feature/volume precision pairlist

config_full.json.example

@@ -49,7 +49,8 @@
         "method": "VolumePairList",
         "config": {
             "number_assets": 20,
-            "sort_key": "quoteVolume"
+            "sort_key": "quoteVolume",
+            "precision_filter": false
         }
     },
     "exchange": {

docs/configuration.md

@@ -280,13 +280,15 @@ By default, a Static Pairlist is used (configured as `"pair_whitelist"` under th
 * `"VolumePairList"`
   * Formerly available as `--dynamic-whitelist [<number_assets>]`
   * Selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`.
+  * Possibility to filter low-value coins that would not allow setting a stop loss
 
 ```json
 "pairlist": {
         "method": "VolumePairList",
         "config": {
             "number_assets": 20,
-            "sort_key": "quoteVolume"
+            "sort_key": "quoteVolume",
+            "precision_filter": false
         }
     },
 ```

freqtrade/freqtradebot.py

@@ -214,7 +214,7 @@ class FreqtradeBot(object):
         """
         return [(pair, self.config['ticker_interval']) for pair in pairs]
 
-    def get_target_bid(self, pair: str) -> float:
+    def get_target_bid(self, pair: str, tick: Dict = None) -> float:
         """
         Calculates bid target between current ask price and last price
         :return: float: Price
@@ -231,8 +231,11 @@ class FreqtradeBot(object):
             logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate)
             used_rate = order_book_rate
         else:
+            if not tick:
                 logger.info('Using Last Ask / Last Price')
                 ticker = self.exchange.get_ticker(pair)
+            else:
+                ticker = tick
             if ticker['ask'] < ticker['last']:
                 ticker_rate = ticker['ask']
             else:

freqtrade/pairlist/IPairList.py

@@ -74,7 +74,7 @@ class IPairList(ABC):
 
         for market in markets:
             pair = market['symbol']
-            # pair is not int the generated dynamic market, or in the blacklist ... ignore it
+            # pair is not in the generated dynamic market, or in the blacklist ... ignore it
             if pair not in whitelist or pair in self.blacklist:
                 continue
             # else the pair is valid

freqtrade/pairlist/VolumePairList.py

@@ -1,5 +1,5 @@
 """
-Static List provider
+Volume PairList provider
 
 Provides lists as configured in config.json
 
@@ -26,6 +26,7 @@ class VolumePairList(IPairList):
                 'for "pairlist.config.number_assets"')
         self._number_pairs = self._whitelistconf['number_assets']
         self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume')
+        self._precision_filter = self._whitelistconf.get('precision_filter', False)
 
         if not self._freqtrade.exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -52,9 +53,9 @@ class VolumePairList(IPairList):
         -> Please overwrite in subclasses
         """
         # Generate dynamic whitelist
-        pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key)
+        self._whitelist = self._gen_pair_whitelist(
-        # Validate whitelist to only have active market pairs
+            self._config['stake_currency'], self._sort_key)[:self._number_pairs]
-        self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs]
+        logger.info(f"Searching pairs: {self._whitelist}")
 
     @cached(TTLCache(maxsize=1, ttl=1800))
     def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]:
@@ -69,7 +70,25 @@ class VolumePairList(IPairList):
         # check length so that we make sure that '/' is actually in the string
         tickers = [v for k, v in tickers.items()
                    if len(k.split('/')) == 2 and k.split('/')[1] == base_currency]
-
         sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key])
-        pairs = [s['symbol'] for s in sorted_tickers]
+        # Validate whitelist to only have active market pairs
+        valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers])
+        valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs]
+
+        if self._freqtrade.strategy.stoploss is not None and self._precision_filter:
+
+            stop_prices = [self._freqtrade.get_target_bid(t["symbol"], t)
+                           * (1 - abs(self._freqtrade.strategy.stoploss)) for t in valid_tickers]
+            rates = [sp * 0.99 for sp in stop_prices]
+            logger.debug("\n".join([f"{sp} : {r}" for sp, r in zip(stop_prices[:10], rates[:10])]))
+            for i, t in enumerate(valid_tickers):
+                sp = self._freqtrade.exchange.symbol_price_prec(t["symbol"], stop_prices[i])
+                r = self._freqtrade.exchange.symbol_price_prec(t["symbol"], rates[i])
+                logger.debug(f"{t['symbol']} - {sp} : {r}")
+                if sp <= r:
+                    logger.info(f"Removed {t['symbol']} from whitelist, "
+                                f"because stop price {sp} would be <= stop limit {r}")
+                    valid_tickers.remove(t)
+
+        pairs = [s['symbol'] for s in valid_tickers]
         return pairs

freqtrade/tests/conftest.py

@@ -375,6 +375,78 @@ def markets():
                 },
             },
             'info': '',
+        },
+        {
+            'id': 'BTTBTC',
+            'symbol': 'BTT/BTC',
+            'base': 'BTT',
+            'quote': 'BTC',
+            'active': True,
+            'precision': {
+                'base': 8,
+                'quote': 8,
+                'amount': 0,
+                'price': 8
+            },
+            'limits': {
+                'amount': {
+                    'min': 1.0,
+                    'max': 90000000.0
+                },
+                'price': {
+                    'min': None,
+                    'max': None
+                },
+                'cost': {
+                    'min': 0.001,
+                    'max': None
+                }
+            },
+            'info': "",
+        },
+        {
+            'id': 'USDT-ETH',
+            'symbol': 'ETH/USDT',
+            'base': 'ETH',
+            'quote': 'USDT',
+            'precision': {
+                'amount': 8,
+                'price': 8
+            },
+            'limits': {
+                'amount': {
+                    'min': 0.02214286,
+                    'max': None
+                },
+                'price': {
+                    'min': 1e-08,
+                    'max': None
+                }
+            },
+            'active': True,
+            'info': ""
+        },
+        {
+            'id': 'USDT-LTC',
+            'symbol': 'LTC/USDT',
+            'base': 'LTC',
+            'quote': 'USDT',
+            'active': True,
+            'precision': {
+                'amount': 8,
+                'price': 8
+            },
+            'limits': {
+                'amount': {
+                    'min': 0.06646786,
+                    'max': None
+                },
+                'price': {
+                    'min': 1e-08,
+                    'max': None
+                }
+            },
+            'info': ""
         }
     ])
 
@@ -595,6 +667,7 @@ def tickers():
             'vwap': 0.01869197,
             'open': 0.018585,
             'close': 0.018573,
+            'last': 0.018799,
             'baseVolume': 81058.66,
             'quoteVolume': 2247.48374509,
         },
@@ -642,6 +715,28 @@ def tickers():
             'quoteVolume': 1401.65697943,
             'info': {}
         },
+        'BTT/BTC': {
+            'symbol': 'BTT/BTC',
+            'timestamp': 1550936557206,
+            'datetime': '2019-02-23T15:42:37.206Z',
+            'high': 0.00000026,
+            'low': 0.00000024,
+            'bid': 0.00000024,
+            'bidVolume': 2446894197.0,
+            'ask': 0.00000025,
+            'askVolume': 2447913837.0,
+            'vwap': 0.00000025,
+            'open': 0.00000026,
+            'close': 0.00000024,
+            'last': 0.00000024,
+            'previousClose': 0.00000026,
+            'change': -0.00000002,
+            'percentage': -7.692,
+            'average': None,
+            'baseVolume': 4886464537.0,
+            'quoteVolume': 1215.14489611,
+            'info': {}
+        },
         'ETH/USDT': {
             'symbol': 'ETH/USDT',
             'timestamp': 1522014804118,

freqtrade/tests/exchange/test_exchange.py

@@ -1309,7 +1309,7 @@ def test_get_markets(default_conf, mocker, markets, exchange_name):
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
     ret = exchange.get_markets()
     assert isinstance(ret, list)
-    assert len(ret) == 6
+    assert len(ret) == 9
 
     assert ret[0]["id"] == "ethbtc"
     assert ret[0]["symbol"] == "ETH/BTC"

freqtrade/tests/pairlist/test_pairlist.py

@@ -80,7 +80,7 @@ def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf):
     freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
 
     # argument: use the whitelist dynamically by exchange-volume
-    whitelist = ['ETH/BTC', 'TKN/BTC']
+    whitelist = ['ETH/BTC', 'TKN/BTC', 'BTT/BTC']
     freqtradebot.pairlists.refresh_pairlist()
 
     assert whitelist == freqtradebot.pairlists.whitelist
@@ -113,23 +113,35 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers)
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_markets', markets)
     mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
+    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8))
 
     # Test to retrieved BTC sorted on quoteVolume (default)
     whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume')
-    assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC']
+    assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BTT/BTC']
 
     # Test to retrieve BTC sorted on bidVolume
     whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume')
-    assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC']
+    assert whitelist == ['BTT/BTC', 'TKN/BTC', 'ETH/BTC']
 
     # Test with USDT sorted on quoteVolume (default)
+    freqtrade.config['stake_currency'] = 'USDT'  # this has to be set, otherwise markets are removed
     whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='USDT', key='quoteVolume')
-    assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT']
+    assert whitelist == ['ETH/USDT', 'LTC/USDT']
 
     # Test with ETH (our fixture does not have ETH, so result should be empty)
+    freqtrade.config['stake_currency'] = 'ETH'
     whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='ETH', key='quoteVolume')
     assert whitelist == []
 
+    freqtrade.pairlists._precision_filter = True
+    freqtrade.config['stake_currency'] = 'BTC'
+    # Retest First 2 test-cases to make sure BTT is not in it (too low priced)
+    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume')
+    assert whitelist == ['ETH/BTC', 'TKN/BTC']
+
+    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume')
+    assert whitelist == ['TKN/BTC', 'ETH/BTC']
+
 
 def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None:
     default_conf['pairlist'] = {'method': 'VolumePairList',