Introduce new test functions to check logs

New functions log_contains, num_log_contains, num_log_has and num_log_has_re
are introduced in the conftest module to help and simplify checking:
- if logs contain a string;
- count how many messages contain a string;
- count how many messages are the given string;
- count how many messages matchs a regex.

A couple of existing tests are changed using the new functions.

tests/conftest.py

@@ -49,6 +49,13 @@ def pytest_configure(config):
         setattr(config.option, 'markexpr', 'not longrun')
 
 
+def log_contains(line, logs, count=False):
+    # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'spamfoobar')
+    # and we want to check if line ('foo', for example) is contained in the tuple.
+    return any(line in logger_name or line in message
+               for logger_name, level, message in logs.record_tuples)
+
+
 def log_has(line, logs):
     # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar')
     # and we want to match line against foobar in the tuple
@@ -63,6 +70,27 @@ def log_has_re(line, logs):
                   False)
 
 
+def num_log_contains(line, logs, count=False):
+    # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'spamfoobar')
+    # and we want to check how many times line ('foo', for example) is contained in the tuples.
+    return sum(line in logger_name or line in message
+               for logger_name, level, message in logs.record_tuples)
+
+
+def num_log_has(line, logs, count=False):
+    # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar')
+    # and we want to check how many times line is presente in the tuples.
+    return sum(line == logger_name or line == message
+               for logger_name, level, message in logs.record_tuples)
+
+
+def num_log_has_re(line, logs, count=False):
+    # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar')
+    # and we want to check how many times line matches in the tuples.
+    return sum(re.match(line, logger_name) or re.match(line, message)
+               for logger_name, level, message in logs.record_tuples)
+
+
 def get_args(args):
     return Arguments(args).get_parsed_arg()
 

tests/exchange/test_exchange.py

@@ -20,7 +20,8 @@ from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes,
                                          timeframe_to_next_date, timeframe_to_prev_date,
                                          timeframe_to_seconds)
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
-from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re
+from tests.conftest import (get_mock_coro, get_patched_exchange, log_contains, log_has, log_has_re,
+                            num_log_contains)
 
 
 # Make sure to always keep one exchange here which is NOT subclassed!!
@@ -1745,8 +1746,8 @@ async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog):
     caplog.set_level(logging.INFO)
     api_mock = MagicMock()
 
-    assert not log_has_re('Kucoin 429 error, avoid triggering DDosProtection backoff delay.*',
+    assert not log_contains('Kucoin 429 error, avoid triggering DDosProtection backoff delay',
-                          caplog)
+                            caplog)
 
     for _ in range(3):
         with pytest.raises(DDosProtection, match=r'429 Too Many Requests'):
@@ -1757,18 +1758,12 @@ async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog):
             exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kucoin")
             await exchange._async_get_candle_history(
                 'ETH/BTC', "5m", (arrow.utcnow().int_timestamp - 2000) * 1000, count=1)
-    logs_found = sum('_async_get_candle_history() returned exception: "kucoin GET ' in message
+    assert num_log_contains('_async_get_candle_history() returned exception: "kucoin GET ',
-                     for message in caplog.messages)
+                            caplog) == 1
-    assert logs_found == 1
+    assert num_log_contains('retrying _async_get_candle_history() still for ', caplog) == 1
-    logs_found = sum('retrying _async_get_candle_history() still for ' in message
+    assert num_log_contains("Kucoin 429 error, avoid triggering DDosProtection backoff delay",
-                     for message in caplog.messages)
+                            caplog) == 1
-    assert logs_found == 1
+    assert num_log_contains('Giving up retrying: _async_get_candle_history()', caplog) == 1
-    logs_found = sum("Kucoin 429 error, avoid triggering DDosProtection backoff delay" in message
-                     for message in caplog.messages)
-    assert logs_found == 1
-    logs_found = sum(message == 'Giving up retrying: _async_get_candle_history()'
-                     for message in caplog.messages)
-    assert logs_found == 1
 
     mocker.patch('freqtrade.exchange.common.calculate_backoff', MagicMock(return_value=0.01))
     for _ in range(3):
@@ -1780,9 +1775,7 @@ async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog):
             exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kucoin")
             await exchange._async_get_candle_history(
                 'ETH/BTC', "5m", (arrow.utcnow().int_timestamp - 2000) * 1000, count=1)
-    logs_found = sum('Applying DDosProtection backoff delay: ' in message
+    assert num_log_contains('Applying DDosProtection backoff delay: ', caplog) == 1
-                     for message in caplog.messages)
-    assert logs_found == 1
 
 
 @pytest.mark.asyncio

tests/plugins/test_pairlist.py

@@ -15,7 +15,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.resolvers import PairListResolver
 from tests.conftest import (create_mock_trades, get_patched_exchange, get_patched_freqtradebot,
-                            log_has, log_has_re)
+                            log_has, log_has_re, num_log_has)
 
 
 @pytest.fixture(scope="function")
@@ -240,16 +240,14 @@ def test_remove_logs_for_pairs_already_in_blacklist(mocker, markets, static_pl_c
     new_whitelist = freqtrade.pairlists.verify_blacklist(whitelist + ['BLK/BTC'], logger.warning)
     # Ensure that the pair is removed from the white list, and properly logged.
     assert set(whitelist) == set(new_whitelist)
-    matches = sum(1 for message in caplog.messages
+    assert num_log_has('Pair BLK/BTC in your blacklist. Removing it from whitelist...',
-                  if message == 'Pair BLK/BTC in your blacklist. Removing it from whitelist...')
+                       caplog) == 1
-    assert matches == 1
 
     new_whitelist = freqtrade.pairlists.verify_blacklist(whitelist + ['BLK/BTC'], logger.warning)
     # Ensure that the pair is not logged anymore when being removed from the pair list.
     assert set(whitelist) == set(new_whitelist)
-    matches = sum(1 for message in caplog.messages
+    assert num_log_has('Pair BLK/BTC in your blacklist. Removing it from whitelist...',
-                  if message == 'Pair BLK/BTC in your blacklist. Removing it from whitelist...')
+                       caplog) == 1
-    assert matches == 1
 
 
 def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf):