Merge branch 'develop' into spice-rack

tests/conftest.py

@@ -58,6 +58,11 @@ def log_has(line, logs):
     return any(line == message for message in logs.messages)
 
 
+def log_has_when(line, logs, when):
+    """Check if line is found in caplog's messages during a specified stage"""
+    return any(line == message.message for message in logs.get_records(when))
+
+
 def log_has_re(line, logs):
     """Check if line matches some caplog's message."""
     return any(re.match(line, message) for message in logs.messages)
@@ -2282,7 +2287,7 @@ def tickers():
 
 
 @pytest.fixture
-def result(testdatadir):
+def dataframe_1m(testdatadir):
     with (testdatadir / 'UNITTEST_BTC-1m.json').open('r') as data_file:
         return ohlcv_to_dataframe(json.load(data_file), '1m', pair="UNITTEST/BTC",
                                   fill_missing=True)

tests/data/test_converter.py

@@ -18,8 +18,8 @@ from tests.conftest import log_has, log_has_re
 from tests.data.test_history import _clean_test_file
 
 
-def test_dataframe_correct_columns(result):
-    assert result.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume']
+def test_dataframe_correct_columns(dataframe_1m):
+    assert dataframe_1m.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume']
 
 
 def test_ohlcv_to_dataframe(ohlcv_history_list, caplog):

tests/data/test_datahandler.py

@@ -0,0 +1,436 @@
+# pragma pylint: disable=missing-docstring, protected-access, C0103
+
+import re
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+from pandas import DataFrame
+
+from freqtrade.configuration import TimeRange
+from freqtrade.constants import AVAILABLE_DATAHANDLERS
+from freqtrade.data.history.featherdatahandler import FeatherDataHandler
+from freqtrade.data.history.hdf5datahandler import HDF5DataHandler
+from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass
+from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
+from freqtrade.data.history.parquetdatahandler import ParquetDataHandler
+from freqtrade.enums import CandleType, TradingMode
+from tests.conftest import log_has
+
+
+def test_datahandler_ohlcv_get_pairs(testdatadir):
+    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC',
+                          'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC',
+                          'DASH/BTC', 'XRP/ETH'}
+
+    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m', candle_type=CandleType.SPOT)
+    assert set(pairs) == {'UNITTEST/BTC'}
+
+    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
+    assert set(pairs) == {'UNITTEST/BTC'}
+
+    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
+    assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'}
+
+    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES)
+    assert set(pairs) == {'XRP/USDT'}
+
+    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
+    assert set(pairs) == {'UNITTEST/USDT:USDT'}
+
+
+@pytest.mark.parametrize('filename,pair,timeframe,candletype', [
+    ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''),
+    ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''),
+    ('BTC-PERP-1h.h5', 'BTC-PERP', '1h', ''),
+    ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''),
+    ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'),
+    ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'),
+    ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'),
+    ('BTC-PERP-1h-index.h5', 'BTC-PERP', '1h', 'index'),
+    ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'),
+])
+def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype):
+    regex = JsonDataHandler._OHLCV_REGEX
+
+    match = re.search(regex, filename)
+    assert len(match.groups()) > 1
+    assert match[1] == pair
+    assert match[2] == timeframe
+    assert match[3] == candletype
+
+
+@pytest.mark.parametrize('input,expected', [
+    ('XMR_USDT', 'XMR/USDT'),
+    ('BTC_USDT', 'BTC/USDT'),
+    ('USDT_BUSD', 'USDT/BUSD'),
+    ('BTC_USDT_USDT', 'BTC/USDT:USDT'),  # Futures
+    ('XRP_USDT_USDT', 'XRP/USDT:USDT'),  # futures
+    ('BTC-PERP', 'BTC-PERP'),
+    ('BTC-PERP_USDT', 'BTC-PERP:USDT'),  # potential FTX case
+    ('UNITTEST_USDT', 'UNITTEST/USDT'),
+])
+def test_rebuild_pair_from_filename(input, expected):
+
+    assert IDataHandler.rebuild_pair_from_filename(input) == expected
+
+
+def test_datahandler_ohlcv_get_available_data(testdatadir):
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {
+        ('UNITTEST/BTC', '5m', CandleType.SPOT),
+        ('ETH/BTC', '5m', CandleType.SPOT),
+        ('XLM/BTC', '5m', CandleType.SPOT),
+        ('TRX/BTC', '5m', CandleType.SPOT),
+        ('LTC/BTC', '5m', CandleType.SPOT),
+        ('XMR/BTC', '5m', CandleType.SPOT),
+        ('ZEC/BTC', '5m', CandleType.SPOT),
+        ('UNITTEST/BTC', '1m', CandleType.SPOT),
+        ('ADA/BTC', '5m', CandleType.SPOT),
+        ('ETC/BTC', '5m', CandleType.SPOT),
+        ('NXT/BTC', '5m', CandleType.SPOT),
+        ('DASH/BTC', '5m', CandleType.SPOT),
+        ('XRP/ETH', '1m', CandleType.SPOT),
+        ('XRP/ETH', '5m', CandleType.SPOT),
+        ('UNITTEST/BTC', '30m', CandleType.SPOT),
+        ('UNITTEST/BTC', '8m', CandleType.SPOT),
+        ('NOPAIR/XXX', '4m', CandleType.SPOT),
+    }
+
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.FUTURES)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {
+        ('UNITTEST/USDT', '1h', 'mark'),
+        ('XRP/USDT', '1h', 'futures'),
+        ('XRP/USDT', '1h', 'mark'),
+        ('XRP/USDT', '8h', 'mark'),
+        ('XRP/USDT', '8h', 'funding_rate'),
+    }
+
+    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {('UNITTEST/BTC', '8m', CandleType.SPOT)}
+    paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)}
+
+
+def test_jsondatahandler_trades_get_pairs(testdatadir):
+    pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
+
+
+def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = JsonGzDataHandler(testdatadir)
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 2
+
+
+def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
+    dh = JsonDataHandler(testdatadir)
+    df = dh.ohlcv_load('XRP/ETH', '5m', 'spot')
+    assert len(df) == 711
+
+    df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark")
+    assert len(df_mark) == 99
+
+    df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot')
+    assert len(df_no_mark) == 0
+
+    # Failure case (empty array)
+    df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', 'spot')
+    assert len(df1) == 0
+    assert log_has("Could not load data for NOPAIR/XXX.", caplog)
+    assert df.columns.equals(df1.columns)
+
+
+@pytest.mark.parametrize('datahandler', ['feather', 'parquet'])
+def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.trades_load('UNITTEST/ETH')
+    with pytest.raises(NotImplementedError):
+        dh.trades_store('UNITTEST/ETH', MagicMock())
+
+
+def test_jsondatahandler_trades_load(testdatadir, caplog):
+    dh = JsonGzDataHandler(testdatadir)
+    logmsg = "Old trades format detected - converting"
+    dh.trades_load('XRP/ETH')
+    assert not log_has(logmsg, caplog)
+
+    # Test conversation is happening
+    dh.trades_load('XRP/OLD')
+    assert log_has(logmsg, caplog)
+
+
+def test_jsondatahandler_trades_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = JsonGzDataHandler(testdatadir)
+    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
+
+
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
+def test_datahandler_ohlcv_append(datahandler, testdatadir, ):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.SPOT)
+    with pytest.raises(NotImplementedError):
+        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.MARK)
+
+
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
+def test_datahandler_trades_append(datahandler, testdatadir):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.trades_append('UNITTEST/ETH', [])
+
+
+def test_hdf5datahandler_trades_get_pairs(testdatadir):
+    pairs = HDF5DataHandler.trades_get_pairs(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'XRP/ETH'}
+
+
+def test_hdf5datahandler_trades_load(testdatadir):
+    dh = get_datahandler(testdatadir, 'hdf5')
+    trades = dh.trades_load('XRP/ETH')
+    assert isinstance(trades, list)
+
+    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    assert trades1 == []
+    # data goes from 2019-10-11 - 2019-10-13
+    timerange = TimeRange.parse_timerange('20191011-20191012')
+
+    trades2 = dh._trades_load('XRP/ETH', timerange)
+    assert len(trades) > len(trades2)
+    # Check that ID is None (If it's nan, it's wrong)
+    assert trades2[0][2] is None
+
+    # unfiltered load has trades before starttime
+    assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0
+    # filtered list does not have trades before starttime
+    assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0
+    # unfiltered load has trades after endtime
+    assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0
+    # filtered list does not have trades after endtime
+    assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0
+
+
+def test_hdf5datahandler_trades_store(testdatadir, tmpdir):
+    tmpdir1 = Path(tmpdir)
+    dh = get_datahandler(testdatadir, 'hdf5')
+    trades = dh.trades_load('XRP/ETH')
+
+    dh1 = get_datahandler(tmpdir1, 'hdf5')
+    dh1.trades_store('XRP/NEW', trades)
+    file = tmpdir1 / 'XRP_NEW-trades.h5'
+    assert file.is_file()
+    # Load trades back
+    trades_new = dh1.trades_load('XRP/NEW')
+
+    assert len(trades_new) == len(trades)
+    assert trades[0][0] == trades_new[0][0]
+    assert trades[0][1] == trades_new[0][1]
+    # assert trades[0][2] == trades_new[0][2]  # This is nan - so comparison does not make sense
+    assert trades[0][3] == trades_new[0][3]
+    assert trades[0][4] == trades_new[0][4]
+    assert trades[0][5] == trades_new[0][5]
+    assert trades[0][6] == trades_new[0][6]
+    assert trades[-1][0] == trades_new[-1][0]
+    assert trades[-1][1] == trades_new[-1][1]
+    # assert trades[-1][2] == trades_new[-1][2]  # This is nan - so comparison does not make sense
+    assert trades[-1][3] == trades_new[-1][3]
+    assert trades[-1][4] == trades_new[-1][4]
+    assert trades[-1][5] == trades_new[-1][5]
+    assert trades[-1][6] == trades_new[-1][6]
+
+
+def test_hdf5datahandler_trades_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
+
+
+@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
+    # Data goes from 2018-01-10 - 2018-01-30
+    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
+    # Mark data goes from to 2021-11-15 2021-11-19
+    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
+])
+def test_hdf5datahandler_ohlcv_load_and_resave(
+    testdatadir,
+    tmpdir,
+    pair,
+    timeframe,
+    candle_type,
+    candle_append,
+    startdt, enddt
+):
+    tmpdir1 = Path(tmpdir)
+    tmpdir2 = tmpdir1
+    if candle_type not in ('', 'spot'):
+        tmpdir2 = tmpdir1 / 'futures'
+        tmpdir2.mkdir()
+    dh = get_datahandler(testdatadir, 'hdf5')
+    ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
+    assert isinstance(ohlcv, DataFrame)
+    assert len(ohlcv) > 0
+
+    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5"
+    assert not file.is_file()
+
+    dh1 = get_datahandler(tmpdir1, 'hdf5')
+    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
+    assert file.is_file()
+
+    assert not ohlcv[ohlcv['date'] < startdt].empty
+
+    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
+
+    # Call private function to ensure timerange is filtered in hdf5
+    ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type)
+    ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
+    assert len(ohlcv) == len(ohlcv1)
+    assert ohlcv.equals(ohlcv1)
+    assert ohlcv[ohlcv['date'] < startdt].empty
+    assert ohlcv[ohlcv['date'] > enddt].empty
+
+    # Try loading inexisting file
+    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
+    assert ohlcv.empty
+
+
+@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
+    # Data goes from 2018-01-10 - 2018-01-30
+    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
+    # Mark data goes from to 2021-11-15 2021-11-19
+    ('UNITTEST/USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
+])
+@pytest.mark.parametrize('datahandler', ['hdf5', 'feather', 'parquet'])
+def test_generic_datahandler_ohlcv_load_and_resave(
+    datahandler,
+    testdatadir,
+    tmpdir,
+    pair,
+    timeframe,
+    candle_type,
+    candle_append,
+    startdt, enddt
+):
+    tmpdir1 = Path(tmpdir)
+    tmpdir2 = tmpdir1
+    if candle_type not in ('', 'spot'):
+        tmpdir2 = tmpdir1 / 'futures'
+        tmpdir2.mkdir()
+    # Load data from one common file
+    dhbase = get_datahandler(testdatadir, 'json')
+    ohlcv = dhbase._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
+    assert isinstance(ohlcv, DataFrame)
+    assert len(ohlcv) > 0
+
+    # Get data to test
+    dh = get_datahandler(testdatadir, datahandler)
+
+    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.{dh._get_file_extension()}"
+    assert not file.is_file()
+
+    dh1 = get_datahandler(tmpdir1, datahandler)
+    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
+    assert file.is_file()
+
+    assert not ohlcv[ohlcv['date'] < startdt].empty
+
+    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
+
+    ohlcv = dhbase.ohlcv_load(pair, timeframe, timerange=timerange, candle_type=candle_type)
+    if datahandler == 'hdf5':
+        ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
+        if candle_type == 'mark':
+            ohlcv1['volume'] = 0.0
+    else:
+        ohlcv1 = dh1.ohlcv_load('UNITTEST/NEW', timeframe,
+                                timerange=timerange, candle_type=candle_type)
+
+    assert len(ohlcv) == len(ohlcv1)
+    assert ohlcv.equals(ohlcv1)
+    assert ohlcv[ohlcv['date'] < startdt].empty
+    assert ohlcv[ohlcv['date'] > enddt].empty
+
+    # Try loading inexisting file
+    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
+    assert ohlcv.empty
+
+
+def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 2
+
+
+def test_gethandlerclass():
+    cl = get_datahandlerclass('json')
+    assert cl == JsonDataHandler
+    assert issubclass(cl, IDataHandler)
+
+    cl = get_datahandlerclass('jsongz')
+    assert cl == JsonGzDataHandler
+    assert issubclass(cl, IDataHandler)
+    assert issubclass(cl, JsonDataHandler)
+
+    cl = get_datahandlerclass('hdf5')
+    assert cl == HDF5DataHandler
+    assert issubclass(cl, IDataHandler)
+
+    cl = get_datahandlerclass('feather')
+    assert cl == FeatherDataHandler
+    assert issubclass(cl, IDataHandler)
+
+    cl = get_datahandlerclass('parquet')
+    assert cl == ParquetDataHandler
+    assert issubclass(cl, IDataHandler)
+
+    with pytest.raises(ValueError, match=r"No datahandler for .*"):
+        get_datahandlerclass('DeadBeef')
+
+
+def test_get_datahandler(testdatadir):
+    dh = get_datahandler(testdatadir, 'json')
+    assert type(dh) == JsonDataHandler
+    dh = get_datahandler(testdatadir, 'jsongz')
+    assert type(dh) == JsonGzDataHandler
+    dh1 = get_datahandler(testdatadir, 'jsongz', dh)
+    assert id(dh1) == id(dh)
+
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert type(dh) == HDF5DataHandler

tests/data/test_dataprovider.py

@@ -144,6 +144,77 @@ def test_available_pairs(mocker, default_conf, ohlcv_history):
     assert dp.available_pairs == [("XRP/BTC", timeframe), ("UNITTEST/BTC", timeframe), ]
 
 
+def test_producer_pairs(mocker, default_conf, ohlcv_history):
+    dataprovider = DataProvider(default_conf, None)
+
+    producer = "default"
+    whitelist = ["XRP/BTC", "ETH/BTC"]
+    assert len(dataprovider.get_producer_pairs(producer)) == 0
+
+    dataprovider._set_producer_pairs(whitelist, producer)
+    assert len(dataprovider.get_producer_pairs(producer)) == 2
+
+    new_whitelist = ["BTC/USDT"]
+    dataprovider._set_producer_pairs(new_whitelist, producer)
+    assert dataprovider.get_producer_pairs(producer) == new_whitelist
+
+    assert dataprovider.get_producer_pairs("bad") == []
+
+
+def test_get_producer_df(mocker, default_conf, ohlcv_history):
+    dataprovider = DataProvider(default_conf, None)
+
+    pair = 'BTC/USDT'
+    timeframe = default_conf['timeframe']
+    candle_type = CandleType.SPOT
+
+    empty_la = datetime.fromtimestamp(0, tz=timezone.utc)
+    now = datetime.now(timezone.utc)
+
+    # no data has been added, any request should return an empty dataframe
+    dataframe, la = dataprovider.get_producer_df(pair, timeframe, candle_type)
+    assert dataframe.empty
+    assert la == empty_la
+
+    # the data is added, should return that added dataframe
+    dataprovider._add_external_df(pair, ohlcv_history, now, timeframe, candle_type)
+    dataframe, la = dataprovider.get_producer_df(pair, timeframe, candle_type)
+    assert len(dataframe) > 0
+    assert la > empty_la
+
+    # no data on this producer, should return empty dataframe
+    dataframe, la = dataprovider.get_producer_df(pair, producer_name='bad')
+    assert dataframe.empty
+    assert la == empty_la
+
+    # non existent timeframe, empty dataframe
+    datframe, la = dataprovider.get_producer_df(pair, timeframe='1h')
+    assert dataframe.empty
+    assert la == empty_la
+
+
+def test_emit_df(mocker, default_conf, ohlcv_history):
+    mocker.patch('freqtrade.rpc.rpc_manager.RPCManager.__init__', MagicMock())
+    rpc_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager', MagicMock())
+    send_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager.send_msg', MagicMock())
+
+    dataprovider = DataProvider(default_conf, exchange=None, rpc=rpc_mock)
+    dataprovider_no_rpc = DataProvider(default_conf, exchange=None)
+
+    pair = "BTC/USDT"
+
+    # No emit yet
+    assert send_mock.call_count == 0
+
+    # Rpc is added, we call emit, should call send_msg
+    dataprovider._emit_df(pair, ohlcv_history)
+    assert send_mock.call_count == 1
+
+    # No rpc added, emit called, should not call send_msg
+    dataprovider_no_rpc._emit_df(pair, ohlcv_history)
+    assert send_mock.call_count == 1
+
+
 def test_refresh(mocker, default_conf, ohlcv_history):
     refresh_mock = MagicMock()
     mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)

tests/data/test_history.py

@@ -1,7 +1,6 @@
 # pragma pylint: disable=missing-docstring, protected-access, C0103
 
 import json
-import re
 import uuid
 from pathlib import Path
 from shutil import copyfile
@@ -13,18 +12,17 @@ from pandas import DataFrame
 from pandas.testing import assert_frame_equal
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import AVAILABLE_DATAHANDLERS, DATETIME_PRINT_FORMAT
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data.converter import ohlcv_to_dataframe
-from freqtrade.data.history.hdf5datahandler import HDF5DataHandler
 from freqtrade.data.history.history_utils import (_download_pair_history, _download_trades_history,
                                                   _load_cached_data_for_updating,
                                                   convert_trades_to_ohlcv, get_timerange, load_data,
                                                   load_pair_history, refresh_backtest_ohlcv_data,
                                                   refresh_backtest_trades_data, refresh_data,
                                                   validate_backtest_data)
-from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass
+from freqtrade.data.history.idatahandler import get_datahandler
 from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
-from freqtrade.enums import CandleType, TradingMode
+from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.resolvers import StrategyResolver
@@ -32,25 +30,6 @@ from tests.conftest import (CURRENT_TEST_STRATEGY, get_patched_exchange, log_has
                             patch_exchange)
 
 
-# Change this if modifying UNITTEST/BTC testdatafile
-_BTC_UNITTEST_LENGTH = 13681
-
-
-def _backup_file(file: Path, copy_file: bool = False) -> None:
-    """
-    Backup existing file to avoid deleting the user file
-    :param file: complete path to the file
-    :param copy_file: keep file in place too.
-    :return: None
-    """
-    file_swp = str(file) + '.swp'
-    if file.is_file():
-        file.rename(file_swp)
-
-        if copy_file:
-            copyfile(file_swp, file)
-
-
 def _clean_test_file(file: Path) -> None:
     """
     Backup existing file to avoid deleting the user file
@@ -67,7 +46,7 @@ def _clean_test_file(file: Path) -> None:
         file_swp.rename(file)
 
 
-def test_load_data_30min_timeframe(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_30min_timeframe(caplog, testdatadir) -> None:
     ld = load_pair_history(pair='UNITTEST/BTC', timeframe='30m', datadir=testdatadir)
     assert isinstance(ld, DataFrame)
     assert not log_has(
@@ -76,7 +55,7 @@ def test_load_data_30min_timeframe(mocker, caplog, default_conf, testdatadir) ->
     )
 
 
-def test_load_data_7min_timeframe(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_7min_timeframe(caplog, testdatadir) -> None:
     ld = load_pair_history(pair='UNITTEST/BTC', timeframe='7m', datadir=testdatadir)
     assert isinstance(ld, DataFrame)
     assert ld.empty
@@ -108,7 +87,7 @@ def test_load_data_mark(ohlcv_history, mocker, caplog, testdatadir) -> None:
     )
 
 
-def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_startup_candles(mocker, testdatadir) -> None:
     ltfmock = mocker.patch(
         'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load',
         MagicMock(return_value=DataFrame()))
@@ -405,7 +384,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None:
                    caplog)
 
 
-def test_init(default_conf, mocker) -> None:
+def test_init(default_conf) -> None:
     assert {} == load_data(
         datadir=Path(''),
         pairs=[],
@@ -685,340 +664,3 @@ def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog):
     convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
                             datadir=tmpdir1, timerange=tr, erase=True)
     assert log_has('Could not convert NoDatapair to OHLCV.', caplog)
-
-
-def test_datahandler_ohlcv_get_pairs(testdatadir):
-    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC',
-                          'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC',
-                          'DASH/BTC', 'XRP/ETH'}
-
-    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m', candle_type=CandleType.SPOT)
-    assert set(pairs) == {'UNITTEST/BTC'}
-
-    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
-    assert set(pairs) == {'UNITTEST/BTC'}
-
-    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'}
-
-    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES)
-    assert set(pairs) == {'XRP/USDT'}
-
-    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT:USDT'}
-
-
-@pytest.mark.parametrize('filename,pair,timeframe,candletype', [
-    ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''),
-    ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''),
-    ('BTC-PERP-1h.h5', 'BTC-PERP', '1h', ''),
-    ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''),
-    ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'),
-    ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'),
-    ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'),
-    ('BTC-PERP-1h-index.h5', 'BTC-PERP', '1h', 'index'),
-    ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'),
-])
-def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype):
-    regex = JsonDataHandler._OHLCV_REGEX
-
-    match = re.search(regex, filename)
-    assert len(match.groups()) > 1
-    assert match[1] == pair
-    assert match[2] == timeframe
-    assert match[3] == candletype
-
-
-@pytest.mark.parametrize('input,expected', [
-    ('XMR_USDT', 'XMR/USDT'),
-    ('BTC_USDT', 'BTC/USDT'),
-    ('USDT_BUSD', 'USDT/BUSD'),
-    ('BTC_USDT_USDT', 'BTC/USDT:USDT'),  # Futures
-    ('XRP_USDT_USDT', 'XRP/USDT:USDT'),  # futures
-    ('BTC-PERP', 'BTC-PERP'),
-    ('BTC-PERP_USDT', 'BTC-PERP:USDT'),  # potential FTX case
-    ('UNITTEST_USDT', 'UNITTEST/USDT'),
-])
-def test_rebuild_pair_from_filename(input, expected):
-
-    assert IDataHandler.rebuild_pair_from_filename(input) == expected
-
-
-def test_datahandler_ohlcv_get_available_data(testdatadir):
-    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    # Convert to set to avoid failures due to sorting
-    assert set(paircombs) == {
-        ('UNITTEST/BTC', '5m', CandleType.SPOT),
-        ('ETH/BTC', '5m', CandleType.SPOT),
-        ('XLM/BTC', '5m', CandleType.SPOT),
-        ('TRX/BTC', '5m', CandleType.SPOT),
-        ('LTC/BTC', '5m', CandleType.SPOT),
-        ('XMR/BTC', '5m', CandleType.SPOT),
-        ('ZEC/BTC', '5m', CandleType.SPOT),
-        ('UNITTEST/BTC', '1m', CandleType.SPOT),
-        ('ADA/BTC', '5m', CandleType.SPOT),
-        ('ETC/BTC', '5m', CandleType.SPOT),
-        ('NXT/BTC', '5m', CandleType.SPOT),
-        ('DASH/BTC', '5m', CandleType.SPOT),
-        ('XRP/ETH', '1m', CandleType.SPOT),
-        ('XRP/ETH', '5m', CandleType.SPOT),
-        ('UNITTEST/BTC', '30m', CandleType.SPOT),
-        ('UNITTEST/BTC', '8m', CandleType.SPOT),
-        ('NOPAIR/XXX', '4m', CandleType.SPOT),
-    }
-
-    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.FUTURES)
-    # Convert to set to avoid failures due to sorting
-    assert set(paircombs) == {
-        ('UNITTEST/USDT', '1h', 'mark'),
-        ('XRP/USDT', '1h', 'futures'),
-        ('XRP/USDT', '1h', 'mark'),
-        ('XRP/USDT', '8h', 'mark'),
-        ('XRP/USDT', '8h', 'funding_rate'),
-    }
-
-    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    assert set(paircombs) == {('UNITTEST/BTC', '8m', CandleType.SPOT)}
-    paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)}
-
-
-def test_jsondatahandler_trades_get_pairs(testdatadir):
-    pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
-
-
-def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = JsonGzDataHandler(testdatadir)
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 2
-
-
-def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
-    dh = JsonDataHandler(testdatadir)
-    df = dh.ohlcv_load('XRP/ETH', '5m', 'spot')
-    assert len(df) == 711
-
-    df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark")
-    assert len(df_mark) == 99
-
-    df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot')
-    assert len(df_no_mark) == 0
-
-    # Failure case (empty array)
-    df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', 'spot')
-    assert len(df1) == 0
-    assert log_has("Could not load data for NOPAIR/XXX.", caplog)
-    assert df.columns.equals(df1.columns)
-
-
-def test_jsondatahandler_trades_load(testdatadir, caplog):
-    dh = JsonGzDataHandler(testdatadir)
-    logmsg = "Old trades format detected - converting"
-    dh.trades_load('XRP/ETH')
-    assert not log_has(logmsg, caplog)
-
-    # Test conversation is happening
-    dh.trades_load('XRP/OLD')
-    assert log_has(logmsg, caplog)
-
-
-def test_jsondatahandler_trades_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = JsonGzDataHandler(testdatadir)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 1
-
-
-@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-def test_datahandler_ohlcv_append(datahandler, testdatadir, ):
-    dh = get_datahandler(testdatadir, datahandler)
-    with pytest.raises(NotImplementedError):
-        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.SPOT)
-    with pytest.raises(NotImplementedError):
-        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.MARK)
-
-
-@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-def test_datahandler_trades_append(datahandler, testdatadir):
-    dh = get_datahandler(testdatadir, datahandler)
-    with pytest.raises(NotImplementedError):
-        dh.trades_append('UNITTEST/ETH', [])
-
-
-def test_hdf5datahandler_trades_get_pairs(testdatadir):
-    pairs = HDF5DataHandler.trades_get_pairs(testdatadir)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH'}
-
-
-def test_hdf5datahandler_trades_load(testdatadir):
-    dh = HDF5DataHandler(testdatadir)
-    trades = dh.trades_load('XRP/ETH')
-    assert isinstance(trades, list)
-
-    trades1 = dh.trades_load('UNITTEST/NONEXIST')
-    assert trades1 == []
-    # data goes from 2019-10-11 - 2019-10-13
-    timerange = TimeRange.parse_timerange('20191011-20191012')
-
-    trades2 = dh._trades_load('XRP/ETH', timerange)
-    assert len(trades) > len(trades2)
-    # Check that ID is None (If it's nan, it's wrong)
-    assert trades2[0][2] is None
-
-    # unfiltered load has trades before starttime
-    assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0
-    # filtered list does not have trades before starttime
-    assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0
-    # unfiltered load has trades after endtime
-    assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0
-    # filtered list does not have trades after endtime
-    assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0
-
-
-def test_hdf5datahandler_trades_store(testdatadir, tmpdir):
-    tmpdir1 = Path(tmpdir)
-    dh = HDF5DataHandler(testdatadir)
-    trades = dh.trades_load('XRP/ETH')
-
-    dh1 = HDF5DataHandler(tmpdir1)
-    dh1.trades_store('XRP/NEW', trades)
-    file = tmpdir1 / 'XRP_NEW-trades.h5'
-    assert file.is_file()
-    # Load trades back
-    trades_new = dh1.trades_load('XRP/NEW')
-
-    assert len(trades_new) == len(trades)
-    assert trades[0][0] == trades_new[0][0]
-    assert trades[0][1] == trades_new[0][1]
-    # assert trades[0][2] == trades_new[0][2]  # This is nan - so comparison does not make sense
-    assert trades[0][3] == trades_new[0][3]
-    assert trades[0][4] == trades_new[0][4]
-    assert trades[0][5] == trades_new[0][5]
-    assert trades[0][6] == trades_new[0][6]
-    assert trades[-1][0] == trades_new[-1][0]
-    assert trades[-1][1] == trades_new[-1][1]
-    # assert trades[-1][2] == trades_new[-1][2]  # This is nan - so comparison does not make sense
-    assert trades[-1][3] == trades_new[-1][3]
-    assert trades[-1][4] == trades_new[-1][4]
-    assert trades[-1][5] == trades_new[-1][5]
-    assert trades[-1][6] == trades_new[-1][6]
-
-
-def test_hdf5datahandler_trades_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = HDF5DataHandler(testdatadir)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 1
-
-
-@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
-    # Data goes from 2018-01-10 - 2018-01-30
-    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
-    # Mark data goes from to 2021-11-15 2021-11-19
-    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
-])
-def test_hdf5datahandler_ohlcv_load_and_resave(
-    testdatadir,
-    tmpdir,
-    pair,
-    timeframe,
-    candle_type,
-    candle_append,
-    startdt, enddt
-):
-    tmpdir1 = Path(tmpdir)
-    tmpdir2 = tmpdir1
-    if candle_type not in ('', 'spot'):
-        tmpdir2 = tmpdir1 / 'futures'
-        tmpdir2.mkdir()
-    dh = HDF5DataHandler(testdatadir)
-    ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
-    assert isinstance(ohlcv, DataFrame)
-    assert len(ohlcv) > 0
-
-    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5"
-    assert not file.is_file()
-
-    dh1 = HDF5DataHandler(tmpdir1)
-    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
-    assert file.is_file()
-
-    assert not ohlcv[ohlcv['date'] < startdt].empty
-
-    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
-
-    # Call private function to ensure timerange is filtered in hdf5
-    ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type)
-    ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
-    assert len(ohlcv) == len(ohlcv1)
-    assert ohlcv.equals(ohlcv1)
-    assert ohlcv[ohlcv['date'] < startdt].empty
-    assert ohlcv[ohlcv['date'] > enddt].empty
-
-    # Try loading inexisting file
-    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
-    assert ohlcv.empty
-
-
-def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = HDF5DataHandler(testdatadir)
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 2
-
-
-def test_gethandlerclass():
-    cl = get_datahandlerclass('json')
-    assert cl == JsonDataHandler
-    assert issubclass(cl, IDataHandler)
-    cl = get_datahandlerclass('jsongz')
-    assert cl == JsonGzDataHandler
-    assert issubclass(cl, IDataHandler)
-    assert issubclass(cl, JsonDataHandler)
-    cl = get_datahandlerclass('hdf5')
-    assert cl == HDF5DataHandler
-    assert issubclass(cl, IDataHandler)
-    with pytest.raises(ValueError, match=r"No datahandler for .*"):
-        get_datahandlerclass('DeadBeef')
-
-
-def test_get_datahandler(testdatadir):
-    dh = get_datahandler(testdatadir, 'json')
-    assert type(dh) == JsonDataHandler
-    dh = get_datahandler(testdatadir, 'jsongz')
-    assert type(dh) == JsonGzDataHandler
-    dh1 = get_datahandler(testdatadir, 'jsongz', dh)
-    assert id(dh1) == id(dh)
-
-    dh = get_datahandler(testdatadir, 'hdf5')
-    assert type(dh) == HDF5DataHandler

tests/exchange/test_exchange.py

@@ -20,6 +20,7 @@ from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_pr
                                 timeframe_to_prev_date, timeframe_to_seconds)
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT,
                                        calculate_backoff, remove_credentials)
+from freqtrade.exchange.exchange import amount_to_contract_precision
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
 from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re
 
@@ -4470,6 +4471,7 @@ def test__amount_to_contracts(
     ('ADA/USDT:USDT', 10.4445555, 10.4, 10.444),
     ('LTC/ETH', 30, 30, 30),
     ('LTC/USD', 30, 30, 30),
+    ('ADA/USDT:USDT', 1.17, 1.1, 1.17),
     # contract size of 10
     ('ETH/USDT:USDT', 10.111, 10.1, 10),
     ('ETH/USDT:USDT', 10.188, 10.1, 10),
@@ -4497,6 +4499,20 @@ def test_amount_to_contract_precision(
     assert result_size == expected_fut
 
 
+@pytest.mark.parametrize('amount,precision,precision_mode,contract_size,expected', [
+    (1.17, 1.0, 4, 0.01, 1.17),  # Tick size
+    (1.17, 1.0, 2, 0.01, 1.17),  #
+    (1.16, 1.0, 4, 0.01, 1.16),  #
+    (1.16, 1.0, 2, 0.01, 1.16),  #
+    (1.13, 1.0, 2, 0.01, 1.13),  #
+    (10.988, 1.0, 2, 10, 10),
+    (10.988, 1.0, 4, 10, 10),
+])
+def test_amount_to_contract_precision2(amount, precision, precision_mode, contract_size, expected):
+    res = amount_to_contract_precision(amount, precision, precision_mode, contract_size)
+    assert pytest.approx(res) == expected
+
+
 @pytest.mark.parametrize('exchange_name,open_rate,is_short,trading_mode,margin_mode', [
     # Bittrex
     ('bittrex', 2.0, False, 'spot', None),

tests/freqai/test_freqai_backtesting.py

@@ -3,21 +3,21 @@ from datetime import datetime, timezone
 from pathlib import Path
 from unittest.mock import PropertyMock
 
-import pytest
-
-from freqtrade.commands.optimize_commands import start_backtesting
-from freqtrade.exceptions import OperationalException
+from freqtrade.commands.optimize_commands import setup_optimize_configuration
+from freqtrade.enums import RunMode
 from freqtrade.optimize.backtesting import Backtesting
 from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
 
-def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir):
+def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, caplog):
     patch_exchange(mocker)
 
+    now = datetime.now(timezone.utc)
     mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
                  PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
-    # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
+    mocker.patch('freqtrade.optimize.backtesting.history.load_data')
+    mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
 
     patched_configuration_load_config_file(mocker, freqai_conf)
 
@@ -30,9 +30,11 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir):
         '--strategy-list', CURRENT_TEST_STRATEGY
     ]
     args = get_args(args)
-    with pytest.raises(OperationalException,
-                       match=r"You can't use strategy_list and freqai at the same time\."):
-        start_backtesting(args)
+    bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
+    Backtesting(bt_config)
+    assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical '
+                      'populate_any_indicators.', caplog)
+    Backtesting.cleanup()
 
 
 def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):

tests/freqai/test_freqai_interface.py

@@ -319,6 +319,41 @@ def test_principal_component_analysis(mocker, freqai_conf):
     shutil.rmtree(Path(freqai.dk.full_path))
 
 
+def test_plot_feature_importance(mocker, freqai_conf):
+
+    from freqtrade.freqai.utils import plot_feature_importance
+
+    freqai_conf.update({"timerange": "20180110-20180130"})
+    freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
+        {"princpial_component_analysis": "true"})
+
+    strategy = get_patched_freqai_strategy(mocker, freqai_conf)
+    exchange = get_patched_exchange(mocker, freqai_conf)
+    strategy.dp = DataProvider(freqai_conf, exchange)
+    strategy.freqai_info = freqai_conf.get("freqai", {})
+    freqai = strategy.freqai
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqai_conf)
+    timerange = TimeRange.parse_timerange("20180110-20180130")
+    freqai.dd.load_all_pair_histories(timerange, freqai.dk)
+
+    freqai.dd.pair_dict = MagicMock()
+
+    data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
+    new_timerange = TimeRange.parse_timerange("20180120-20180130")
+
+    freqai.extract_data_and_train_model(
+        new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
+
+    model = freqai.dd.load_data("ADA/BTC", freqai.dk)
+
+    plot_feature_importance(model, "ADA/BTC", freqai.dk)
+
+    assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}.html")
+
+    shutil.rmtree(Path(freqai.dk.full_path))
+
+
 def test_spice_rack(mocker, default_conf, tmpdir, caplog):
 
     strategy = get_patched_freqai_strategy(mocker, default_conf)

tests/optimize/test_optimize_reports.py

@@ -1,6 +1,7 @@
 import re
 from datetime import timedelta
 from pathlib import Path
+from shutil import copyfile
 
 import joblib
 import pandas as pd
@@ -25,7 +26,22 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene
                                                  text_table_exit_reason, text_table_strategy)
 from freqtrade.resolvers.strategy_resolver import StrategyResolver
 from tests.conftest import CURRENT_TEST_STRATEGY
-from tests.data.test_history import _backup_file, _clean_test_file
+from tests.data.test_history import _clean_test_file
+
+
+def _backup_file(file: Path, copy_file: bool = False) -> None:
+    """
+    Backup existing file to avoid deleting the user file
+    :param file: complete path to the file
+    :param copy_file: keep file in place too.
+    :return: None
+    """
+    file_swp = str(file) + '.swp'
+    if file.is_file():
+        file.rename(file_swp)
+
+        if copy_file:
+            copyfile(file_swp, file)
 
 
 def test_text_table_bt_results():

tests/plugins/test_pairlist.py

@@ -467,6 +467,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
       {"method": "RangeStabilityFilter", "lookback_days": 10,
        "max_rate_of_change": 0.01, "refresh_period": 1440}],
      "BTC", []),  # All removed because of max_rate_of_change being 0.017
+    ([{"method": "StaticPairList"},
+      {"method": "RangeStabilityFilter", "lookback_days": 10,
+        "min_rate_of_change": 0.018, "max_rate_of_change": 0.02, "refresh_period": 1440}],
+     "BTC", []),  # All removed - limits are above the highest change_rate
     ([{"method": "StaticPairList"},
       {"method": "VolatilityFilter", "lookback_days": 3,
        "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}],

tests/rpc/test_rpc_apiserver.py

@@ -3,6 +3,8 @@ Unit test file for rpc/api_server.py
 """
 
 import json
+import logging
+import time
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from unittest.mock import ANY, MagicMock, PropertyMock
@@ -10,7 +12,7 @@ from unittest.mock import ANY, MagicMock, PropertyMock
 import pandas as pd
 import pytest
 import uvicorn
-from fastapi import FastAPI
+from fastapi import FastAPI, WebSocketDisconnect
 from fastapi.exceptions import HTTPException
 from fastapi.testclient import TestClient
 from requests.auth import _basic_auth_str
@@ -31,6 +33,7 @@ from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_mock_
 BASE_URI = "/api/v1"
 _TEST_USER = "FreqTrader"
 _TEST_PASS = "SuperSecurePassword1!"
+_TEST_WS_TOKEN = "secret_Ws_t0ken"
 
 
 @pytest.fixture
@@ -44,17 +47,21 @@ def botclient(default_conf, mocker):
                                         "CORS_origins": ['http://example.com'],
                                         "username": _TEST_USER,
                                         "password": _TEST_PASS,
+                                        "ws_token": _TEST_WS_TOKEN
                                         }})
 
     ftbot = get_patched_freqtradebot(mocker, default_conf)
     rpc = RPC(ftbot)
     mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api', MagicMock())
+    apiserver = None
     try:
         apiserver = ApiServer(default_conf)
         apiserver.add_rpc_handler(rpc)
         yield ftbot, TestClient(apiserver.app)
         # Cleanup ... ?
     finally:
+        if apiserver:
+            apiserver.cleanup()
         ApiServer.shutdown()
 
 
@@ -154,6 +161,25 @@ def test_api_auth():
         get_user_from_token(b'not_a_token', 'secret1234')
 
 
+def test_api_ws_auth(botclient):
+    ftbot, client = botclient
+    def url(token): return f"/api/v1/message/ws?token={token}"
+
+    bad_token = "bad-ws_token"
+    with pytest.raises(WebSocketDisconnect):
+        with client.websocket_connect(url(bad_token)) as websocket:
+            websocket.receive()
+
+    good_token = _TEST_WS_TOKEN
+    with client.websocket_connect(url(good_token)) as websocket:
+        pass
+
+    jwt_secret = ftbot.config['api_server'].get('jwt_secret_key', 'super-secret')
+    jwt_token = create_token({'identity': {'u': 'Freqtrade'}}, jwt_secret)
+    with client.websocket_connect(url(jwt_token)) as websocket:
+        pass
+
+
 def test_api_unauthorized(botclient):
     ftbot, client = botclient
     rc = client.get(f"{BASE_URI}/ping")
@@ -261,6 +287,7 @@ def test_api__init__(default_conf, mocker):
     with pytest.raises(OperationalException, match="RPC Handler already attached."):
         apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf)))
 
+    apiserver.cleanup()
     ApiServer.shutdown()
 
 
@@ -388,6 +415,7 @@ def test_api_run(default_conf, mocker, caplog):
                  MagicMock(side_effect=Exception))
     apiserver.start_api()
     assert log_has("Api server failed to start.", caplog)
+    apiserver.cleanup()
     ApiServer.shutdown()
 
 
@@ -410,6 +438,7 @@ def test_api_cleanup(default_conf, mocker, caplog):
     apiserver.cleanup()
     assert apiserver._server.cleanup.call_count == 1
     assert log_has("Stopping API Server", caplog)
+    assert log_has("Stopping API Server background tasks", caplog)
     ApiServer.shutdown()
 
 
@@ -1663,3 +1692,93 @@ def test_health(botclient):
     ret = rc.json()
     assert ret['last_process_ts'] == 0
     assert ret['last_process'] == '1970-01-01T00:00:00+00:00'
+
+
+def test_api_ws_subscribe(botclient, mocker):
+    ftbot, client = botclient
+    ws_url = f"/api/v1/message/ws?token={_TEST_WS_TOKEN}"
+
+    sub_mock = mocker.patch('freqtrade.rpc.api_server.ws.WebSocketChannel.set_subscriptions')
+
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({'type': 'subscribe', 'data': ['whitelist']})
+
+    # Check call count is now 1 as we sent a valid subscribe request
+    assert sub_mock.call_count == 1
+
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({'type': 'subscribe', 'data': 'whitelist'})
+
+    # Call count hasn't changed as the subscribe request was invalid
+    assert sub_mock.call_count == 1
+
+
+def test_api_ws_requests(botclient, mocker, caplog):
+    caplog.set_level(logging.DEBUG)
+
+    ftbot, client = botclient
+    ws_url = f"/api/v1/message/ws?token={_TEST_WS_TOKEN}"
+
+    # Test whitelist request
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({"type": "whitelist", "data": None})
+        response = ws.receive_json()
+
+    assert log_has_re(r"Request of type whitelist from.+", caplog)
+    assert response['type'] == "whitelist"
+
+    # Test analyzed_df request
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({"type": "analyzed_df", "data": {}})
+        response = ws.receive_json()
+
+    assert log_has_re(r"Request of type analyzed_df from.+", caplog)
+    assert response['type'] == "analyzed_df"
+
+    caplog.clear()
+    # Test analyzed_df request with data
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({"type": "analyzed_df", "data": {"limit": 100}})
+        response = ws.receive_json()
+
+    assert log_has_re(r"Request of type analyzed_df from.+", caplog)
+    assert response['type'] == "analyzed_df"
+
+
+def test_api_ws_send_msg(default_conf, mocker, caplog):
+    try:
+        caplog.set_level(logging.DEBUG)
+
+        default_conf.update({"api_server": {"enabled": True,
+                                            "listen_ip_address": "127.0.0.1",
+                                            "listen_port": 8080,
+                                            "CORS_origins": ['http://example.com'],
+                                            "username": _TEST_USER,
+                                            "password": _TEST_PASS,
+                                            "ws_token": _TEST_WS_TOKEN
+                                            }})
+        mocker.patch('freqtrade.rpc.telegram.Updater')
+        mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api')
+        apiserver = ApiServer(default_conf)
+        apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf)))
+        apiserver.start_message_queue()
+        # Give the queue thread time to start
+        time.sleep(0.2)
+
+        # Test message_queue coro receives the message
+        test_message = {"type": "status", "data": "test"}
+        apiserver.send_msg(test_message)
+        time.sleep(0.1)  # Not sure how else to wait for the coro to receive the data
+        assert log_has("Found message of type: status", caplog)
+
+        # Test if exception logged when error occurs in sending
+        mocker.patch('freqtrade.rpc.api_server.ws.channel.ChannelManager.broadcast',
+                     side_effect=Exception)
+
+        apiserver.send_msg(test_message)
+        time.sleep(0.1)  # Not sure how else to wait for the coro to receive the data
+        assert log_has_re(r"Exception happened in background task.*", caplog)
+
+    finally:
+        apiserver.cleanup()
+        ApiServer.shutdown()

tests/rpc/test_rpc_emc.py

@@ -0,0 +1,465 @@
+"""
+Unit test file for rpc/external_message_consumer.py
+"""
+import asyncio
+import functools
+import logging
+from datetime import datetime, timezone
+from unittest.mock import MagicMock
+
+import pytest
+import websockets
+
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
+from tests.conftest import log_has, log_has_re, log_has_when
+
+
+_TEST_WS_TOKEN = "secret_Ws_t0ken"
+_TEST_WS_HOST = "127.0.0.1"
+_TEST_WS_PORT = 9989
+
+
+@pytest.fixture
+def patched_emc(default_conf, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "null",
+                    "port": 9891,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ]
+        }
+    })
+    dataprovider = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dataprovider)
+
+    try:
+        yield emc
+    finally:
+        emc.shutdown()
+
+
+def test_emc_start(patched_emc, caplog):
+    # Test if the message was printed
+    assert log_has_when("Starting ExternalMessageConsumer", caplog, "setup")
+    # Test if the thread and loop objects were created
+    assert patched_emc._thread and patched_emc._loop
+
+    # Test we call start again nothing happens
+    prev_thread = patched_emc._thread
+    patched_emc.start()
+    assert prev_thread == patched_emc._thread
+
+
+def test_emc_shutdown(patched_emc, caplog):
+    patched_emc.shutdown()
+
+    assert log_has("Stopping ExternalMessageConsumer", caplog)
+    # Test the loop has stopped
+    assert patched_emc._loop is None
+    # Test if the thread has stopped
+    assert patched_emc._thread is None
+
+    caplog.clear()
+    patched_emc.shutdown()
+
+    # Test func didn't run again as it was called once already
+    assert not log_has("Stopping ExternalMessageConsumer", caplog)
+
+
+def test_emc_init(patched_emc):
+    # Test the settings were set correctly
+    assert patched_emc.initial_candle_limit <= 1500
+    assert patched_emc.wait_timeout > 0
+    assert patched_emc.sleep_time > 0
+
+
+# Parametrize this?
+def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):
+    test_producer = {"name": "test", "url": "ws://test", "ws_token": "test"}
+    producer_name = test_producer['name']
+
+    caplog.set_level(logging.DEBUG)
+
+    # Test handle whitelist message
+    whitelist_message = {"type": "whitelist", "data": ["BTC/USDT"]}
+    patched_emc.handle_producer_message(test_producer, whitelist_message)
+
+    assert log_has(f"Received message of type `whitelist` from `{producer_name}`", caplog)
+    assert log_has(
+        f"Consumed message from `{producer_name}` of type `RPCMessageType.WHITELIST`", caplog)
+
+    # Test handle analyzed_df message
+    df_message = {
+        "type": "analyzed_df",
+        "data": {
+            "key": ("BTC/USDT", "5m", "spot"),
+            "df": ohlcv_history,
+            "la": datetime.now(timezone.utc)
+        }
+    }
+    patched_emc.handle_producer_message(test_producer, df_message)
+
+    assert log_has(f"Received message of type `analyzed_df` from `{producer_name}`", caplog)
+    assert log_has(
+        f"Consumed message from `{producer_name}` of type `RPCMessageType.ANALYZED_DF`", caplog)
+
+    # Test unhandled message
+    unhandled_message = {"type": "status", "data": "RUNNING"}
+    patched_emc.handle_producer_message(test_producer, unhandled_message)
+
+    assert log_has_re(r"Received unhandled message\: .*", caplog)
+
+    # Test malformed messages
+    caplog.clear()
+    malformed_message = {"type": "whitelist", "data": {"pair": "BTC/USDT"}}
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has_re(r"Invalid message .+", caplog)
+
+    malformed_message = {
+        "type": "analyzed_df",
+        "data": {
+            "key": "BTC/USDT",
+            "df": ohlcv_history,
+            "la": datetime.now(timezone.utc)
+        }
+    }
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has(f"Received message of type `analyzed_df` from `{producer_name}`", caplog)
+    assert log_has_re(r"Invalid message .+", caplog)
+
+    caplog.clear()
+    malformed_message = {"some": "stuff"}
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has_re(r"Invalid message .+", caplog)
+
+    caplog.clear()
+    malformed_message = {"type": "whitelist", "data": None}
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has_re(r"Empty message .+", caplog)
+
+
+async def test_emc_create_connection_success(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+    lock = asyncio.Lock()
+
+    emc._running = True
+
+    async def eat(websocket):
+        emc._running = False
+
+    try:
+        async with websockets.serve(eat, _TEST_WS_HOST, _TEST_WS_PORT):
+            await emc._create_connection(test_producer, lock)
+
+        assert log_has_re(r"Producer connection success.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_create_connection_invalid_port(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": -1,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    try:
+        await asyncio.sleep(0.01)
+        assert log_has_re(r".+ is an invalid WebSocket URL .+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_create_connection_invalid_host(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "10000.1241..2121/",
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    try:
+        await asyncio.sleep(0.01)
+        assert log_has_re(r".+ is an invalid WebSocket URL .+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_create_connection_error(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    # Test unexpected error
+    mocker.patch('websockets.connect', side_effect=RuntimeError)
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    try:
+        await asyncio.sleep(0.01)
+        assert log_has("Unexpected error has occurred:", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_valid(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 1,
+            "ping_timeout": 60,
+            "sleep_time": 60
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            return {"type": "whitelist", "data": ["BTC/USDT"]}
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Received message of type `whitelist`.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_invalid(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 1,
+            "ping_timeout": 60,
+            "sleep_time": 60
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            return {"type": ["BTC/USDT"]}
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Invalid message from.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_timeout(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 0.1,
+            "ping_timeout": 1,
+            "sleep_time": 1
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            await asyncio.sleep(0.2)
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Ping error.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_handle_error(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 1,
+            "ping_timeout": 1,
+            "sleep_time": 1
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    emc.handle_producer_message = MagicMock(side_effect=Exception)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            return {"type": "whitelist", "data": ["BTC/USDT"]}
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Error handling producer message.+", caplog)
+    finally:
+        emc.shutdown()

tests/rpc/test_rpc_manager.py

@@ -82,6 +82,21 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None:
     assert telegram_mock.call_count == 0
 
 
+def test_send_msg_telegram_error(mocker, default_conf, caplog) -> None:
+    mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
+    mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', side_effect=ValueError())
+
+    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    rpc_manager = RPCManager(freqtradebot)
+    rpc_manager.send_msg({
+        'type': RPCMessageType.STATUS,
+        'status': 'test'
+    })
+
+    assert log_has("Sending rpc message: {'type': status, 'status': 'test'}", caplog)
+    assert log_has("Exception occurred within RPC module telegram", caplog)
+
+
 def test_process_msg_queue(mocker, default_conf, caplog) -> None:
     telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg')
     mocker.patch('freqtrade.rpc.telegram.Telegram._init')

tests/rpc/test_rpc_telegram.py

@@ -959,6 +959,7 @@ def test_telegram_forceexit_handle(default_conf, update, ticker, fee,
         'gain': 'profit',
         'leverage': 1.0,
         'limit': 1.173e-05,
+        'order_rate': 1.173e-05,
         'amount': 91.07468123,
         'order_type': 'limit',
         'open_rate': 1.098e-05,
@@ -1031,6 +1032,7 @@ def test_telegram_force_exit_down_handle(default_conf, update, ticker, fee,
         'gain': 'loss',
         'leverage': 1.0,
         'limit': 1.043e-05,
+        'order_rate': 1.043e-05,
         'amount': 91.07468123,
         'order_type': 'limit',
         'open_rate': 1.098e-05,
@@ -1092,6 +1094,7 @@ def test_forceexit_all_handle(default_conf, update, ticker, fee, mocker) -> None
         'pair': 'ETH/BTC',
         'gain': 'loss',
         'leverage': 1.0,
+        'order_rate': 1.099e-05,
         'limit': 1.099e-05,
         'amount': 91.07468123,
         'order_type': 'limit',
@@ -1744,7 +1747,7 @@ def test_send_msg_enter_notification(default_conf, mocker, caplog, message_type,
         'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'leverage': leverage,
-        'limit': 1.099e-05,
+        'open_rate': 1.099e-05,
         'order_type': 'limit',
         'direction': enter,
         'stake_amount': 0.01465333,
@@ -1915,7 +1918,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
         'leverage': 1.0,
         'direction': 'Long',
         'gain': 'loss',
-        'limit': 3.201e-05,
+        'order_rate': 3.201e-05,
         'amount': 1333.3333333333335,
         'order_type': 'market',
         'open_rate': 7.5e-05,
@@ -1950,7 +1953,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
         'pair': 'KEY/ETH',
         'direction': 'Long',
         'gain': 'loss',
-        'limit': 3.201e-05,
+        'order_rate': 3.201e-05,
         'amount': 1333.3333333333335,
         'order_type': 'market',
         'open_rate': 7.5e-05,
@@ -1989,7 +1992,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
         'pair': 'KEY/ETH',
         'direction': 'Long',
         'gain': 'loss',
-        'limit': 3.201e-05,
+        'order_rate': 3.201e-05,
         'amount': 1333.3333333333335,
         'order_type': 'market',
         'open_rate': 7.5e-05,
@@ -2162,7 +2165,7 @@ def test_send_msg_buy_notification_no_fiat(
         'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'leverage': leverage,
-        'limit': 1.099e-05,
+        'open_rate': 1.099e-05,
         'order_type': 'limit',
         'direction': enter,
         'stake_amount': 0.01465333,
@@ -2205,7 +2208,7 @@ def test_send_msg_sell_notification_no_fiat(
         'gain': 'loss',
         'leverage': leverage,
         'direction': direction,
-        'limit': 3.201e-05,
+        'order_rate': 3.201e-05,
         'amount': 1333.3333333333335,
         'order_type': 'limit',
         'open_rate': 7.5e-05,

tests/strategy/test_default_strategy.py

@@ -21,14 +21,14 @@ def test_strategy_test_v3_structure():
     (True, 'short'),
     (False, 'long'),
 ])
-def test_strategy_test_v3(result, fee, is_short, side):
+def test_strategy_test_v3(dataframe_1m, fee, is_short, side):
     strategy = StrategyTestV3({})
 
     metadata = {'pair': 'ETH/BTC'}
     assert type(strategy.minimal_roi) is dict
     assert type(strategy.stoploss) is float
     assert type(strategy.timeframe) is str
-    indicators = strategy.populate_indicators(result, metadata)
+    indicators = strategy.populate_indicators(dataframe_1m, metadata)
     assert type(indicators) is DataFrame
     assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame
     assert type(strategy.populate_sell_trend(indicators, metadata)) is DataFrame

tests/strategy/test_strategy_loading.py

@@ -53,7 +53,7 @@ def test_search_all_strategies_with_failed():
     assert len(strategies) == 0
 
 
-def test_load_strategy(default_conf, result):
+def test_load_strategy(default_conf, dataframe_1m):
     default_conf.update({'strategy': 'SampleStrategy',
                          'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates')
                          })
@@ -61,22 +61,22 @@ def test_load_strategy(default_conf, result):
     assert isinstance(strategy.__source__, str)
     assert 'class SampleStrategy' in strategy.__source__
     assert isinstance(strategy.__file__, str)
-    assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
+    assert 'rsi' in strategy.advise_indicators(dataframe_1m, {'pair': 'ETH/BTC'})
 
 
-def test_load_strategy_base64(result, caplog, default_conf):
+def test_load_strategy_base64(dataframe_1m, caplog, default_conf):
     filepath = Path(__file__).parents[2] / 'freqtrade/templates/sample_strategy.py'
     encoded_string = urlsafe_b64encode(filepath.read_bytes()).decode("utf-8")
     default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)})
 
     strategy = StrategyResolver.load_strategy(default_conf)
-    assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
+    assert 'rsi' in strategy.advise_indicators(dataframe_1m, {'pair': 'ETH/BTC'})
     # Make sure strategy was loaded from base64 (using temp directory)!!
     assert log_has_re(r"Using resolved strategy SampleStrategy from '"
                       r".*(/|\\).*(/|\\)SampleStrategy\.py'\.\.\.", caplog)
 
 
-def test_load_strategy_invalid_directory(result, caplog, default_conf):
+def test_load_strategy_invalid_directory(caplog, default_conf):
     default_conf['strategy'] = 'StrategyTestV3'
     extra_dir = Path.cwd() / 'some/path'
     with pytest.raises(OperationalException):
@@ -104,7 +104,7 @@ def test_load_strategy_noname(default_conf):
 
 @pytest.mark.filterwarnings("ignore:deprecated")
 @pytest.mark.parametrize('strategy_name', ['StrategyTestV2'])
-def test_strategy_pre_v3(result, default_conf, strategy_name):
+def test_strategy_pre_v3(dataframe_1m, default_conf, strategy_name):
     default_conf.update({'strategy': strategy_name})
 
     strategy = StrategyResolver.load_strategy(default_conf)
@@ -118,7 +118,7 @@ def test_strategy_pre_v3(result, default_conf, strategy_name):
     assert strategy.timeframe == '5m'
     assert default_conf['timeframe'] == '5m'
 
-    df_indicators = strategy.advise_indicators(result, metadata=metadata)
+    df_indicators = strategy.advise_indicators(dataframe_1m, metadata=metadata)
     assert 'adx' in df_indicators
 
     dataframe = strategy.advise_entry(df_indicators, metadata=metadata)
@@ -417,24 +417,24 @@ def test_call_deprecated_function(default_conf):
         StrategyResolver.load_strategy(default_conf)
 
 
-def test_strategy_interface_versioning(result, default_conf):
+def test_strategy_interface_versioning(dataframe_1m, default_conf):
     default_conf.update({'strategy': 'StrategyTestV2'})
     strategy = StrategyResolver.load_strategy(default_conf)
     metadata = {'pair': 'ETH/BTC'}
 
     assert strategy.INTERFACE_VERSION == 2
 
-    indicator_df = strategy.advise_indicators(result, metadata=metadata)
+    indicator_df = strategy.advise_indicators(dataframe_1m, metadata=metadata)
     assert isinstance(indicator_df, DataFrame)
     assert 'adx' in indicator_df.columns
 
-    enterdf = strategy.advise_entry(result, metadata=metadata)
+    enterdf = strategy.advise_entry(dataframe_1m, metadata=metadata)
     assert isinstance(enterdf, DataFrame)
 
     assert 'buy' not in enterdf.columns
     assert 'enter_long' in enterdf.columns
 
-    exitdf = strategy.advise_exit(result, metadata=metadata)
+    exitdf = strategy.advise_exit(dataframe_1m, metadata=metadata)
     assert isinstance(exitdf, DataFrame)
     assert 'sell' not in exitdf
     assert 'exit_long' in exitdf

tests/test_configuration.py

@@ -1089,6 +1089,58 @@ def test__validate_pricing_rules(default_conf, caplog) -> None:
         validate_config_consistency(conf)
 
 
+def test__validate_consumers(default_conf, caplog) -> None:
+    conf = deepcopy(default_conf)
+    conf.update({
+            "external_message_consumer": {
+                "enabled": True,
+                "producers": []
+                }
+            })
+    with pytest.raises(OperationalException,
+                       match="You must specify at least 1 Producer to connect to."):
+        validate_config_consistency(conf)
+
+    conf = deepcopy(default_conf)
+    conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8081,
+                    "ws_token": "secret_ws_t0ken."
+                }, {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8080,
+                    "ws_token": "secret_ws_t0ken."
+                }
+            ]}
+        })
+    with pytest.raises(OperationalException,
+                       match="Producer names must be unique. Duplicate: default"):
+        validate_config_consistency(conf)
+
+    conf = deepcopy(default_conf)
+    conf.update({
+        "process_only_new_candles": True,
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8081,
+                    "ws_token": "secret_ws_t0ken."
+                }
+            ]}
+        })
+    validate_config_consistency(conf)
+    assert log_has_re("To receive best performance with external data.*", caplog)
+
+
 def test_load_config_test_comments() -> None:
     """
     Load config with comments

tests/test_freqtradebot.py

@@ -1319,9 +1319,9 @@ def test_create_stoploss_order_invalid_order(
     assert create_order_mock.call_args[1]['amount'] == trade.amount
 
     # Rpc is sending first buy, then sell
-    assert rpc_mock.call_count == 2
-    assert rpc_mock.call_args_list[1][0][0]['sell_reason'] == ExitType.EMERGENCY_EXIT.value
-    assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market'
+    assert rpc_mock.call_count == 3
+    assert rpc_mock.call_args_list[2][0][0]['sell_reason'] == ExitType.EMERGENCY_EXIT.value
+    assert rpc_mock.call_args_list[2][0][0]['order_type'] == 'market'
 
 
 @pytest.mark.parametrize("is_short", [False, True])
@@ -2439,7 +2439,7 @@ def test_manage_open_orders_entry_usercustom(
     # Trade should be closed since the function returns true
     freqtrade.manage_open_orders()
     assert cancel_order_wr_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 0
@@ -2478,7 +2478,7 @@ def test_manage_open_orders_entry(
     # check it does cancel buy orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 0
@@ -2608,7 +2608,7 @@ def test_check_handle_cancelled_buy(
     # check it does cancel buy orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 0
     assert log_has_re(
@@ -2639,7 +2639,7 @@ def test_manage_open_orders_buy_exception(
     # check it does cancel buy orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 0
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 1
@@ -2686,7 +2686,7 @@ def test_manage_open_orders_exit_usercustom(
     # Return false - No impact
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 0
+    assert rpc_mock.call_count == 1
     assert open_trade_usdt.is_open is False
     assert freqtrade.strategy.check_exit_timeout.call_count == 1
     assert freqtrade.strategy.check_entry_timeout.call_count == 0
@@ -2696,7 +2696,7 @@ def test_manage_open_orders_exit_usercustom(
     # Return Error - No impact
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 0
+    assert rpc_mock.call_count == 1
     assert open_trade_usdt.is_open is False
     assert freqtrade.strategy.check_exit_timeout.call_count == 1
     assert freqtrade.strategy.check_entry_timeout.call_count == 0
@@ -2706,7 +2706,7 @@ def test_manage_open_orders_exit_usercustom(
     freqtrade.strategy.check_entry_timeout = MagicMock(return_value=True)
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     assert open_trade_usdt.is_open is True
     assert freqtrade.strategy.check_exit_timeout.call_count == 1
     assert freqtrade.strategy.check_entry_timeout.call_count == 0
@@ -2766,7 +2766,7 @@ def test_manage_open_orders_exit(
     # check it does cancel sell orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     assert open_trade_usdt.is_open is True
     # Custom user sell-timeout is never called
     assert freqtrade.strategy.check_exit_timeout.call_count == 0
@@ -2805,7 +2805,7 @@ def test_check_handle_cancelled_exit(
     # check it does cancel sell orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     assert open_trade_usdt.is_open is True
     exit_name = 'Buy' if is_short else 'Sell'
     assert log_has_re(f"{exit_name} order cancelled on exchange for Trade.*", caplog)
@@ -2843,7 +2843,7 @@ def test_manage_open_orders_partial(
     # note this is for a partially-complete buy order
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     assert trades[0].amount == 23.0
@@ -2890,7 +2890,7 @@ def test_manage_open_orders_partial_fee(
     assert log_has_re(r"Applying fee on amount for Trade.*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2940,7 +2940,7 @@ def test_manage_open_orders_partial_except(
     assert log_has_re(r"Could not update trade amount: .*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -3155,7 +3155,7 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None:
     reason = CANCEL_REASON['TIMEOUT']
     assert freqtrade.handle_cancel_exit(trade, order, reason)
     assert cancel_order_mock.call_count == 1
-    assert send_msg_mock.call_count == 1
+    assert send_msg_mock.call_count == 2
     assert trade.close_rate is None
     assert trade.exit_reason is None
 
@@ -3256,6 +3256,7 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_
         'pair': 'ETH/USDT',
         'gain': 'profit',
         'limit': 2.0 if is_short else 2.2,
+        'order_rate': 2.0 if is_short else 2.2,
         'amount': pytest.approx(amt),
         'order_type': 'limit',
         'buy_tag': None,
@@ -3321,6 +3322,7 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd
         'leverage': 1.0,
         'gain': 'loss',
         'limit': 2.2 if is_short else 2.01,
+        'order_rate': 2.2 if is_short else 2.01,
         'amount': pytest.approx(29.70297029) if is_short else 30.0,
         'order_type': 'limit',
         'buy_tag': None,
@@ -3405,6 +3407,7 @@ def test_execute_trade_exit_custom_exit_price(
         'leverage': 1.0,
         'gain': profit_or_loss,
         'limit': limit,
+        'order_rate': limit,
         'amount': pytest.approx(amount),
         'order_type': 'limit',
         'buy_tag': None,
@@ -3476,6 +3479,7 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(
         'leverage': 1.0,
         'gain': 'loss',
         'limit': 2.02 if is_short else 1.98,
+        'order_rate': 2.02 if is_short else 1.98,
         'amount': pytest.approx(29.70297029 if is_short else 30.0),
         'order_type': 'limit',
         'buy_tag': None,
@@ -3588,7 +3592,7 @@ def test_execute_trade_exit_with_stoploss_on_exchange(
     trade.is_short = is_short
     assert trade
     assert cancel_order.call_count == 1
-    assert rpc_mock.call_count == 3
+    assert rpc_mock.call_count == 4
 
 
 @pytest.mark.parametrize("is_short", [False, True])
@@ -3658,11 +3662,11 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit(
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
     assert trade.exit_reason == ExitType.STOPLOSS_ON_EXCHANGE.value
-    assert rpc_mock.call_count == 3
-    assert rpc_mock.call_args_list[0][0][0]['type'] == RPCMessageType.ENTRY
-    assert rpc_mock.call_args_list[0][0][0]['amount'] > 20
-    assert rpc_mock.call_args_list[1][0][0]['type'] == RPCMessageType.ENTRY_FILL
-    assert rpc_mock.call_args_list[2][0][0]['type'] == RPCMessageType.EXIT_FILL
+    assert rpc_mock.call_count == 4
+    assert rpc_mock.call_args_list[1][0][0]['type'] == RPCMessageType.ENTRY
+    assert rpc_mock.call_args_list[1][0][0]['amount'] > 20
+    assert rpc_mock.call_args_list[2][0][0]['type'] == RPCMessageType.ENTRY_FILL
+    assert rpc_mock.call_args_list[3][0][0]['type'] == RPCMessageType.EXIT_FILL
 
 
 @pytest.mark.parametrize(
@@ -3741,6 +3745,7 @@ def test_execute_trade_exit_market_order(
         'leverage': 1.0,
         'gain': profit_or_loss,
         'limit': limit,
+        'order_rate': limit,
         'amount': pytest.approx(amount),
         'order_type': 'market',
         'buy_tag': None,

tests/test_integration.py

@@ -521,4 +521,4 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog) -> Non
     assert trade.orders[-1].ft_order_side == 'sell'
     assert pytest.approx(trade.stake_amount) == 40.198
     assert trade.is_open
-    assert log_has_re('Amount to sell is 0.0 due to exchange limits - not selling.', caplog)
+    assert log_has_re('Amount to exit is 0.0 due to exchange limits - not exiting.', caplog)

tests/test_misc.py

@@ -7,10 +7,11 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.misc import (decimals_per_coin, deep_merge_dicts, file_dump_json, file_load_json,
-                            format_ms_time, pair_to_filename, parse_db_uri_for_logging, plural,
-                            render_template, render_template_with_fallback, round_coin_value,
-                            safe_value_fallback, safe_value_fallback2, shorten_date)
+from freqtrade.misc import (dataframe_to_json, decimals_per_coin, deep_merge_dicts, file_dump_json,
+                            file_load_json, format_ms_time, json_to_dataframe, pair_to_filename,
+                            parse_db_uri_for_logging, plural, render_template,
+                            render_template_with_fallback, round_coin_value, safe_value_fallback,
+                            safe_value_fallback2, shorten_date)
 
 
 def test_decimals_per_coin():
@@ -184,8 +185,8 @@ def test_render_template_fallback(mocker):
             templatefile='subtemplates/indicators_does-not-exist.j2',)
 
     val = render_template_with_fallback(
-        templatefile='subtemplates/indicators_does-not-exist.j2',
-        templatefallbackfile='subtemplates/indicators_minimal.j2',
+        templatefile='strategy_subtemplates/indicators_does-not-exist.j2',
+        templatefallbackfile='strategy_subtemplates/indicators_minimal.j2',
     )
     assert isinstance(val, str)
     assert 'if self.dp' in val
@@ -219,3 +220,14 @@ def test_deep_merge_dicts():
 
     res2['first']['rows']['test'] = 'asdf'
     assert deep_merge_dicts(a, deepcopy(b), allow_null_overrides=False) == res2
+
+
+def test_dataframe_json(ohlcv_history):
+    from pandas.testing import assert_frame_equal
+    json = dataframe_to_json(ohlcv_history)
+    dataframe = json_to_dataframe(json)
+
+    assert list(ohlcv_history.columns) == list(dataframe.columns)
+    assert len(ohlcv_history) == len(dataframe)
+
+    assert_frame_equal(ohlcv_history, dataframe)