Add volumePairList - refactor tests to correct file

freqtrade/freqtradebot.py

@@ -12,8 +12,6 @@ from typing import Any, Callable, Dict, List, Optional
 import arrow
 from requests.exceptions import RequestException
 
-from cachetools import TTLCache, cached
-
 from freqtrade import (DependencyException, OperationalException,
                        TemporaryError, __version__, constants, persistence)
 from freqtrade.exchange import Exchange
@@ -25,7 +23,8 @@ from freqtrade.resolvers import StrategyResolver
 from freqtrade.state import State
 from freqtrade.strategy.interface import SellType, IStrategy
 from freqtrade.exchange.exchange_helpers import order_book_to_dataframe
-from freqtrade.pairlist.StaticList import StaticList
+from freqtrade.pairlist.StaticPairList import StaticPairList
+from freqtrade.pairlist.VolumePairList import VolumePairList
 
 
 logger = logging.getLogger(__name__)
@@ -60,7 +59,11 @@ class FreqtradeBot(object):
         self.persistence = None
         self.exchange = Exchange(self.config)
         self.wallets = Wallets(self.exchange)
-        self.pairlists = StaticList(self, self.config)
+        if self.config.get('dynamic_whitelist', None):
+            self.pairlists = VolumePairList(self, self.config)
+
+        else:
+            self.pairlists = StaticPairList(self, self.config)
 
         # Initializing Edge only if enabled
         self.edge = Edge(self.config, self.exchange, self.strategy) if \
@@ -148,18 +151,9 @@ class FreqtradeBot(object):
         """
         state_changed = False
         try:
-            nb_assets = self.config.get('dynamic_whitelist', None)
+            # Refresh whitelist
-            # Refresh whitelist based on wallet maintenance
             self.pairlists.refresh_whitelist()
-            sanitized_list = self.pairlists.whitelist
+            self.active_pair_whitelist = self.pairlists.whitelist
-            # sanitized_list = self._refresh_whitelist(
-            #     self._gen_pair_whitelist(
-            #         self.config['stake_currency']
-            #     ) if nb_assets else self.lists.get_whitelist()
-            # )
-
-            # Keep only the subsets of pairs wanted (up to nb_assets)
-            self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list
 
             # Calculating Edge positiong
             # Should be called before refresh_tickers
@@ -207,30 +201,6 @@ class FreqtradeBot(object):
             self.state = State.STOPPED
         return state_changed
 
-    @cached(TTLCache(maxsize=1, ttl=1800))
-    def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]:
-        """
-        Updates the whitelist with with a dynamically generated list
-        :param base_currency: base currency as str
-        :param key: sort key (defaults to 'quoteVolume')
-        :return: List of pairs
-        """
-
-        if not self.exchange.exchange_has('fetchTickers'):
-            raise OperationalException(
-                'Exchange does not support dynamic whitelist.'
-                'Please edit your config and restart the bot'
-            )
-
-        tickers = self.exchange.get_tickers()
-        # check length so that we make sure that '/' is actually in the string
-        tickers = [v for k, v in tickers.items()
-                   if len(k.split('/')) == 2 and k.split('/')[1] == base_currency]
-
-        sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key])
-        pairs = [s['symbol'] for s in sorted_tickers]
-        return pairs
-
     def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float:
         """
         Calculates bid target between current ask price and last price

freqtrade/pairlist/StaticPairList.py

@@ -10,7 +10,7 @@ from typing import List, Optional
 logger = logging.getLogger(__name__)
 
 
-class StaticList(object):
+class StaticPairList(object):
 
     def __init__(self, freqtrade, config: dict) -> None:
         self._freqtrade = freqtrade
@@ -32,9 +32,9 @@ class StaticList(object):
         """
         Refreshes whitelist and assigns it to self._whitelist
         """
-        self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist'])
+        self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist'])
 
-    def validate_whitelist(self, whitelist: List[str]) -> List[str]:
+    def _validate_whitelist(self, whitelist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary
         :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to

freqtrade/pairlist/VolumePairList.py

@@ -0,0 +1,98 @@
+"""
+Static List provider
+
+Provides lists as configured in config.json
+
+ """
+import logging
+from typing import List
+from cachetools import TTLCache, cached
+
+from freqtrade.pairlist.StaticPairList import StaticPairList
+from freqtrade import OperationalException
+logger = logging.getLogger(__name__)
+
+
+class VolumePairList(StaticPairList):
+
+    def __init__(self, freqtrade, config: dict) -> None:
+        self._freqtrade = freqtrade
+        self._config = config
+        self._whitelist = self._config['exchange']['pair_whitelist']
+        self._blacklist = self._config['exchange'].get('pair_blacklist', [])
+        self._number_pairs = self._config.get('dynamic_whitelist', None)
+        if not self._freqtrade.exchange.exchange_has('fetchTickers'):
+            raise OperationalException(
+                'Exchange does not support dynamic whitelist.'
+                'Please edit your config and restart the bot'
+            )
+        # self.refresh_whitelist()
+
+    @property
+    def whitelist(self) -> List[str]:
+        """ Contains the current whitelist """
+        return self._whitelist
+
+    @property
+    def blacklist(self) -> List[str]:
+        return self._blacklist
+
+    def refresh_whitelist(self) -> None:
+        """
+        Refreshes whitelist and assigns it to self._whitelist
+        """
+        # Generate dynamic whitelist
+        pairs = self._gen_pair_whitelist(self._config['stake_currency'])
+        # Validate whitelist to only have active market pairs
+        self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs]
+
+    @cached(TTLCache(maxsize=1, ttl=1800))
+    def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]:
+        """
+        Updates the whitelist with with a dynamically generated list
+        :param base_currency: base currency as str
+        :param key: sort key (defaults to 'quoteVolume')
+        :return: List of pairs
+        """
+
+        tickers = self._freqtrade.exchange.get_tickers()
+        # check length so that we make sure that '/' is actually in the string
+        tickers = [v for k, v in tickers.items()
+                   if len(k.split('/')) == 2 and k.split('/')[1] == base_currency]
+
+        sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key])
+        pairs = [s['symbol'] for s in sorted_tickers]
+        return pairs
+
+    def _validate_whitelist(self, whitelist: List[str]) -> List[str]:
+        """
+        Check available markets and remove pair from whitelist if necessary
+        :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to
+        trade
+        :return: the list of pairs the user wants to trade without the one unavailable or
+        black_listed
+        """
+        sanitized_whitelist = whitelist
+        markets = self._freqtrade.exchange.get_markets()
+
+        # Filter to markets in stake currency
+        markets = [m for m in markets if m['quote'] == self._config['stake_currency']]
+        known_pairs = set()
+
+        for market in markets:
+            pair = market['symbol']
+            # pair is not int the generated dynamic market, or in the blacklist ... ignore it
+            if pair not in whitelist or pair in self.blacklist:
+                continue
+            # else the pair is valid
+            known_pairs.add(pair)
+            # Market is not active
+            if not market['active']:
+                sanitized_whitelist.remove(pair)
+                logger.info(
+                    'Ignoring %s from whitelist. Market is not active.',
+                    pair
+                )
+
+        # We need to remove pairs that are unknown
+        return [x for x in sanitized_whitelist if x in known_pairs]

freqtrade/tests/test_acl_pair.py

@@ -2,6 +2,7 @@
 
 from unittest.mock import MagicMock
 
+from freqtrade import OperationalException
 from freqtrade.tests.conftest import get_patched_freqtradebot
 
 import pytest
@@ -35,7 +36,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf):
     # List ordered by BaseVolume
     whitelist = ['ETH/BTC', 'TKN/BTC']
     # Ensure all except those in whitelist are removed
-    assert whitelist == freqtradebot.pairlists.whitelist
+    assert set(whitelist) == set(freqtradebot.pairlists.whitelist)
     # Ensure config dict hasn't been changed
     assert (whitelist_conf['exchange']['pair_whitelist'] ==
             freqtradebot.config['exchange']['pair_whitelist'])
@@ -49,11 +50,12 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf):
     # List ordered by BaseVolume
     whitelist = ['ETH/BTC', 'TKN/BTC']
     # Ensure all except those in whitelist are removed
-    assert whitelist == freqtradebot.pairlists.whitelist
+    assert set(whitelist)== set(freqtradebot.pairlists.whitelist)
     assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist
 
 
 def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf):
+    whitelist_conf['dynamic_whitelist'] = 5
     freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -64,10 +66,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf):
 
     # argument: use the whitelist dynamically by exchange-volume
     whitelist = ['ETH/BTC', 'TKN/BTC']
-
+    freqtradebot.pairlists.refresh_whitelist()
-    freqtradebot._refresh_whitelist(
-        freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency'])
-    )
 
     assert whitelist == freqtradebot.pairlists.whitelist
 
@@ -79,7 +78,40 @@ def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf):
     # argument: use the whitelist dynamically by exchange-volume
     whitelist = []
     whitelist_conf['exchange']['pair_whitelist'] = []
-    freqtradebot._refresh_whitelist(whitelist)
+    freqtradebot.pairlists.refresh_whitelist()
     pairslist = whitelist_conf['exchange']['pair_whitelist']
 
     assert set(whitelist) == set(pairslist)
+
+
+def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None:
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    # Test to retrieved BTC sorted on quoteVolume (default)
+    freqtrade.pairlists.refresh_whitelist()
+
+    whitelist = freqtrade.pairlists.whitelist
+    assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC']
+
+    # Test to retrieve BTC sorted on bidVolume
+    whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume')
+    assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC']
+
+    # Test with USDT sorted on quoteVolume (default)
+    whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT')
+    assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT']
+
+    # Test with ETH (our fixture does not have ETH, so result should be empty)
+    whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH')
+    assert whitelist == []
+
+
+def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None:
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False))
+
+    with pytest.raises(OperationalException):
+        freqtrade._gen_pair_whitelist(base_currency='BTC')

freqtrade/tests/test_freqtradebot.py

@@ -136,37 +136,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None:
     assert result == -1
 
 
-def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None:
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
-
-    # Test to retrieved BTC sorted on quoteVolume (default)
-    whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC')
-    assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC']
-
-    # Test to retrieve BTC sorted on bidVolume
-    whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume')
-    assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC']
-
-    # Test with USDT sorted on quoteVolume (default)
-    whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT')
-    assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT']
-
-    # Test with ETH (our fixture does not have ETH, so result should be empty)
-    whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH')
-    assert whitelist == []
-
-
-def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None:
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False))
-
-    with pytest.raises(OperationalException):
-        freqtrade._gen_pair_whitelist(base_currency='BTC')
-
-
 def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)