Merge pull request #4110 from freqtrade/test/exchange_ccxt

add tests to verify exchange compatibility with ccxt

.github/workflows/ci.yml

@@ -61,6 +61,12 @@ jobs:
     - name: Tests
       run: |
         pytest --random-order --cov=freqtrade --cov-config=.coveragerc
+      if: matrix.python-version != '3.9'
+
+    - name: Tests incl. ccxt compatibility tests
+      run: |
+        pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun
+      if: matrix.python-version == '3.9'
 
     - name: Coveralls
       if: (startsWith(matrix.os, 'ubuntu-20') && matrix.python-version == '3.8')

docs/developer.md

@@ -242,6 +242,9 @@ The `IProtection` parent class provides a helper method for this in `calculate_l
 
 Most exchanges supported by CCXT should work out of the box.
 
+To quickly test the public endpoints of an exchange, add a configuration for your exchange to `test_ccxt_compat.py` and run these tests with `pytest --longrun tests/exchange/test_ccxt_compat.py`.
+Completing these tests successfully a good basis point (it's a requirement, actually), however these won't guarantee correct exchange functioning, as this only tests public endpoints, but no private endpoint (like generate order or similar).
+
 ### Stoploss On Exchange
 
 Check if the new exchange supports Stoploss on Exchange orders through their API.

tests/conftest.py

@@ -33,6 +33,19 @@ logging.getLogger('').setLevel(logging.INFO)
 np.seterr(all='raise')
 
 
+def pytest_addoption(parser):
+    parser.addoption('--longrun', action='store_true', dest="longrun",
+                     default=False, help="Enable long-run tests (ccxt compat)")
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "longrun: mark test that is running slowly and should not be run regularily"
+    )
+    if not config.option.longrun:
+        setattr(config.option, 'markexpr', 'not longrun')
+
+
 def log_has(line, logs):
     # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar')
     # and we want to match line against foobar in the tuple
@@ -224,6 +237,10 @@ def init_persistence(default_conf):
 
 @pytest.fixture(scope="function")
 def default_conf(testdatadir):
+    return get_default_conf(testdatadir)
+
+
+def get_default_conf(testdatadir):
     """ Returns validated configuration suitable for most tests """
     configuration = {
         "max_open_trades": 1,

tests/exchange/test_ccxt_compat.py

@@ -0,0 +1,134 @@
+"""
+Tests in this file do NOT mock network calls, so they are expected to be fluky at times.
+
+However, these tests should give a good idea to determine if a new exchange is
+suitable to run with freqtrade.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from freqtrade.resolvers.exchange_resolver import ExchangeResolver
+from tests.conftest import get_default_conf
+
+
+# Exchanges that should be tested
+EXCHANGES = {
+    'bittrex': {
+        'pair': 'BTC/USDT',
+        'hasQuoteVolume': False,
+        'timeframe': '5m',
+    },
+    'binance': {
+        'pair': 'BTC/USDT',
+        'hasQuoteVolume': True,
+        'timeframe': '5m',
+    },
+    'kraken': {
+        'pair': 'BTC/USDT',
+        'hasQuoteVolume': True,
+        'timeframe': '5m',
+    },
+    'ftx': {
+        'pair': 'BTC/USDT',
+        'hasQuoteVolume': True,
+        'timeframe': '5m',
+    }
+}
+
+
+@pytest.fixture(scope="class")
+def exchange_conf():
+    config = get_default_conf((Path(__file__).parent / "testdata").resolve())
+    config['exchange']['pair_whitelist'] = []
+    return config
+
+
+@pytest.fixture(params=EXCHANGES, scope="class")
+def exchange(request, exchange_conf):
+    exchange_conf['exchange']['name'] = request.param
+    exchange = ExchangeResolver.load_exchange(request.param, exchange_conf, validate=True)
+
+    yield exchange, request.param
+
+
+@pytest.mark.longrun
+class TestCCXTExchange():
+
+    def test_load_markets(self, exchange):
+        exchange, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+        markets = exchange.markets
+        assert pair in markets
+        assert isinstance(markets[pair], dict)
+
+    def test_ccxt_fetch_tickers(self, exchange):
+        exchange, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+
+        tickers = exchange.get_tickers()
+        assert pair in tickers
+        assert 'ask' in tickers[pair]
+        assert tickers[pair]['ask'] is not None
+        assert 'bid' in tickers[pair]
+        assert tickers[pair]['bid'] is not None
+        assert 'quoteVolume' in tickers[pair]
+        if EXCHANGES[exchangename].get('hasQuoteVolume'):
+            assert tickers[pair]['quoteVolume'] is not None
+
+    def test_ccxt_fetch_ticker(self, exchange):
+        exchange, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+
+        ticker = exchange.fetch_ticker(pair)
+        assert 'ask' in ticker
+        assert ticker['ask'] is not None
+        assert 'bid' in ticker
+        assert ticker['bid'] is not None
+        assert 'quoteVolume' in ticker
+        if EXCHANGES[exchangename].get('hasQuoteVolume'):
+            assert ticker['quoteVolume'] is not None
+
+    def test_ccxt_fetch_l2_orderbook(self, exchange):
+        exchange, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+        l2 = exchange.fetch_l2_order_book(pair)
+        assert 'asks' in l2
+        assert 'bids' in l2
+        l2_limit_range = exchange._ft_has['l2_limit_range']
+        for val in [1, 2, 5, 25, 100]:
+            l2 = exchange.fetch_l2_order_book(pair, val)
+            if not l2_limit_range or val in l2_limit_range:
+                assert len(l2['asks']) == val
+                assert len(l2['bids']) == val
+            else:
+                next_limit = exchange.get_next_limit_in_list(val, l2_limit_range)
+                if next_limit > 200:
+                    # Large orderbook sizes can be a problem for some exchanges (bitrex ...)
+                    assert len(l2['asks']) > 200
+                    assert len(l2['asks']) > 200
+                else:
+                    assert len(l2['asks']) == next_limit
+                    assert len(l2['asks']) == next_limit
+
+    def test_fetch_ohlcv(self, exchange):
+        exchange, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+        timeframe = EXCHANGES[exchangename]['timeframe']
+        pair_tf = (pair, timeframe)
+        ohlcv = exchange.refresh_latest_ohlcv([pair_tf])
+        assert isinstance(ohlcv, dict)
+        assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf))
+        assert len(exchange.klines(pair_tf)) > 200
+
+    # TODO: tests fetch_trades (?)
+
+    def test_ccxt_get_fee(self, exchange):
+        exchange, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+
+        assert 0 < exchange.get_fee(pair, 'limit', 'buy') < 1
+        assert 0 < exchange.get_fee(pair, 'limit', 'sell') < 1
+        assert 0 < exchange.get_fee(pair, 'market', 'buy') < 1
+        assert 0 < exchange.get_fee(pair, 'market', 'sell') < 1