Merge remote-tracking branch 'origin/develop' into dev-merge-rl

tests/conftest.py

@@ -58,6 +58,11 @@ def log_has(line, logs):
     return any(line == message for message in logs.messages)
 
 
+def log_has_when(line, logs, when):
+    """Check if line is found in caplog's messages during a specified stage"""
+    return any(line == message.message for message in logs.get_records(when))
+
+
 def log_has_re(line, logs):
     """Check if line matches some caplog's message."""
     return any(re.match(line, message) for message in logs.messages)
@@ -195,6 +200,8 @@ def patch_freqtradebot(mocker, config) -> None:
     mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock())
     mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock())
     patch_whitelist(mocker, config)
+    mocker.patch('freqtrade.freqtradebot.ExternalMessageConsumer')
+    mocker.patch('freqtrade.configuration.config_validation._validate_consumers')
 
 
 def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:

tests/data/test_btanalysis.py

@@ -235,7 +235,7 @@ def test_calculate_market_change(testdatadir):
     data = load_data(datadir=testdatadir, pairs=pairs, timeframe='5m')
     result = calculate_market_change(data)
     assert isinstance(result, float)
-    assert pytest.approx(result) == 0.00955514
+    assert pytest.approx(result) == 0.01100002
 
 
 def test_combine_dataframes_with_mean(testdatadir):
@@ -275,7 +275,7 @@ def test_create_cum_profit1(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     # Move close-time to "off" the candle, to make sure the logic still works
-    bt_data.loc[:, 'close_date'] = bt_data.loc[:, 'close_date'] + DateOffset(seconds=20)
+    bt_data['close_date'] = bt_data.loc[:, 'close_date'] + DateOffset(seconds=20)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
     df = load_pair_history(pair="TRX/BTC", timeframe='5m',

tests/data/test_datahandler.py

@@ -0,0 +1,436 @@
+# pragma pylint: disable=missing-docstring, protected-access, C0103
+
+import re
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+from pandas import DataFrame
+
+from freqtrade.configuration import TimeRange
+from freqtrade.constants import AVAILABLE_DATAHANDLERS
+from freqtrade.data.history.featherdatahandler import FeatherDataHandler
+from freqtrade.data.history.hdf5datahandler import HDF5DataHandler
+from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass
+from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
+from freqtrade.data.history.parquetdatahandler import ParquetDataHandler
+from freqtrade.enums import CandleType, TradingMode
+from tests.conftest import log_has
+
+
+def test_datahandler_ohlcv_get_pairs(testdatadir):
+    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC',
+                          'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC',
+                          'DASH/BTC', 'XRP/ETH'}
+
+    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m', candle_type=CandleType.SPOT)
+    assert set(pairs) == {'UNITTEST/BTC'}
+
+    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
+    assert set(pairs) == {'UNITTEST/BTC'}
+
+    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
+    assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'}
+
+    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES)
+    assert set(pairs) == {'XRP/USDT'}
+
+    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
+    assert set(pairs) == {'UNITTEST/USDT:USDT'}
+
+
+@pytest.mark.parametrize('filename,pair,timeframe,candletype', [
+    ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''),
+    ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''),
+    ('BTC-PERP-1h.h5', 'BTC-PERP', '1h', ''),
+    ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''),
+    ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'),
+    ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'),
+    ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'),
+    ('BTC-PERP-1h-index.h5', 'BTC-PERP', '1h', 'index'),
+    ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'),
+])
+def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype):
+    regex = JsonDataHandler._OHLCV_REGEX
+
+    match = re.search(regex, filename)
+    assert len(match.groups()) > 1
+    assert match[1] == pair
+    assert match[2] == timeframe
+    assert match[3] == candletype
+
+
+@pytest.mark.parametrize('input,expected', [
+    ('XMR_USDT', 'XMR/USDT'),
+    ('BTC_USDT', 'BTC/USDT'),
+    ('USDT_BUSD', 'USDT/BUSD'),
+    ('BTC_USDT_USDT', 'BTC/USDT:USDT'),  # Futures
+    ('XRP_USDT_USDT', 'XRP/USDT:USDT'),  # futures
+    ('BTC-PERP', 'BTC-PERP'),
+    ('BTC-PERP_USDT', 'BTC-PERP:USDT'),  # potential FTX case
+    ('UNITTEST_USDT', 'UNITTEST/USDT'),
+])
+def test_rebuild_pair_from_filename(input, expected):
+
+    assert IDataHandler.rebuild_pair_from_filename(input) == expected
+
+
+def test_datahandler_ohlcv_get_available_data(testdatadir):
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {
+        ('UNITTEST/BTC', '5m', CandleType.SPOT),
+        ('ETH/BTC', '5m', CandleType.SPOT),
+        ('XLM/BTC', '5m', CandleType.SPOT),
+        ('TRX/BTC', '5m', CandleType.SPOT),
+        ('LTC/BTC', '5m', CandleType.SPOT),
+        ('XMR/BTC', '5m', CandleType.SPOT),
+        ('ZEC/BTC', '5m', CandleType.SPOT),
+        ('UNITTEST/BTC', '1m', CandleType.SPOT),
+        ('ADA/BTC', '5m', CandleType.SPOT),
+        ('ETC/BTC', '5m', CandleType.SPOT),
+        ('NXT/BTC', '5m', CandleType.SPOT),
+        ('DASH/BTC', '5m', CandleType.SPOT),
+        ('XRP/ETH', '1m', CandleType.SPOT),
+        ('XRP/ETH', '5m', CandleType.SPOT),
+        ('UNITTEST/BTC', '30m', CandleType.SPOT),
+        ('UNITTEST/BTC', '8m', CandleType.SPOT),
+        ('NOPAIR/XXX', '4m', CandleType.SPOT),
+    }
+
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.FUTURES)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {
+        ('UNITTEST/USDT', '1h', 'mark'),
+        ('XRP/USDT', '1h', 'futures'),
+        ('XRP/USDT', '1h', 'mark'),
+        ('XRP/USDT', '8h', 'mark'),
+        ('XRP/USDT', '8h', 'funding_rate'),
+    }
+
+    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {('UNITTEST/BTC', '8m', CandleType.SPOT)}
+    paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)}
+
+
+def test_jsondatahandler_trades_get_pairs(testdatadir):
+    pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
+
+
+def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = JsonGzDataHandler(testdatadir)
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 2
+
+
+def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
+    dh = JsonDataHandler(testdatadir)
+    df = dh.ohlcv_load('XRP/ETH', '5m', 'spot')
+    assert len(df) == 712
+
+    df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark")
+    assert len(df_mark) == 100
+
+    df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot')
+    assert len(df_no_mark) == 0
+
+    # Failure case (empty array)
+    df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', 'spot')
+    assert len(df1) == 0
+    assert log_has("Could not load data for NOPAIR/XXX.", caplog)
+    assert df.columns.equals(df1.columns)
+
+
+@pytest.mark.parametrize('datahandler', ['feather', 'parquet'])
+def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.trades_load('UNITTEST/ETH')
+    with pytest.raises(NotImplementedError):
+        dh.trades_store('UNITTEST/ETH', MagicMock())
+
+
+def test_jsondatahandler_trades_load(testdatadir, caplog):
+    dh = JsonGzDataHandler(testdatadir)
+    logmsg = "Old trades format detected - converting"
+    dh.trades_load('XRP/ETH')
+    assert not log_has(logmsg, caplog)
+
+    # Test conversation is happening
+    dh.trades_load('XRP/OLD')
+    assert log_has(logmsg, caplog)
+
+
+def test_jsondatahandler_trades_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = JsonGzDataHandler(testdatadir)
+    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
+
+
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
+def test_datahandler_ohlcv_append(datahandler, testdatadir, ):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.SPOT)
+    with pytest.raises(NotImplementedError):
+        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.MARK)
+
+
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
+def test_datahandler_trades_append(datahandler, testdatadir):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.trades_append('UNITTEST/ETH', [])
+
+
+def test_hdf5datahandler_trades_get_pairs(testdatadir):
+    pairs = HDF5DataHandler.trades_get_pairs(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'XRP/ETH'}
+
+
+def test_hdf5datahandler_trades_load(testdatadir):
+    dh = get_datahandler(testdatadir, 'hdf5')
+    trades = dh.trades_load('XRP/ETH')
+    assert isinstance(trades, list)
+
+    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    assert trades1 == []
+    # data goes from 2019-10-11 - 2019-10-13
+    timerange = TimeRange.parse_timerange('20191011-20191012')
+
+    trades2 = dh._trades_load('XRP/ETH', timerange)
+    assert len(trades) > len(trades2)
+    # Check that ID is None (If it's nan, it's wrong)
+    assert trades2[0][2] is None
+
+    # unfiltered load has trades before starttime
+    assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0
+    # filtered list does not have trades before starttime
+    assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0
+    # unfiltered load has trades after endtime
+    assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0
+    # filtered list does not have trades after endtime
+    assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0
+
+
+def test_hdf5datahandler_trades_store(testdatadir, tmpdir):
+    tmpdir1 = Path(tmpdir)
+    dh = get_datahandler(testdatadir, 'hdf5')
+    trades = dh.trades_load('XRP/ETH')
+
+    dh1 = get_datahandler(tmpdir1, 'hdf5')
+    dh1.trades_store('XRP/NEW', trades)
+    file = tmpdir1 / 'XRP_NEW-trades.h5'
+    assert file.is_file()
+    # Load trades back
+    trades_new = dh1.trades_load('XRP/NEW')
+
+    assert len(trades_new) == len(trades)
+    assert trades[0][0] == trades_new[0][0]
+    assert trades[0][1] == trades_new[0][1]
+    # assert trades[0][2] == trades_new[0][2]  # This is nan - so comparison does not make sense
+    assert trades[0][3] == trades_new[0][3]
+    assert trades[0][4] == trades_new[0][4]
+    assert trades[0][5] == trades_new[0][5]
+    assert trades[0][6] == trades_new[0][6]
+    assert trades[-1][0] == trades_new[-1][0]
+    assert trades[-1][1] == trades_new[-1][1]
+    # assert trades[-1][2] == trades_new[-1][2]  # This is nan - so comparison does not make sense
+    assert trades[-1][3] == trades_new[-1][3]
+    assert trades[-1][4] == trades_new[-1][4]
+    assert trades[-1][5] == trades_new[-1][5]
+    assert trades[-1][6] == trades_new[-1][6]
+
+
+def test_hdf5datahandler_trades_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
+
+
+@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
+    # Data goes from 2018-01-10 - 2018-01-30
+    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
+    # Mark data goes from to 2021-11-15 2021-11-19
+    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
+])
+def test_hdf5datahandler_ohlcv_load_and_resave(
+    testdatadir,
+    tmpdir,
+    pair,
+    timeframe,
+    candle_type,
+    candle_append,
+    startdt, enddt
+):
+    tmpdir1 = Path(tmpdir)
+    tmpdir2 = tmpdir1
+    if candle_type not in ('', 'spot'):
+        tmpdir2 = tmpdir1 / 'futures'
+        tmpdir2.mkdir()
+    dh = get_datahandler(testdatadir, 'hdf5')
+    ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
+    assert isinstance(ohlcv, DataFrame)
+    assert len(ohlcv) > 0
+
+    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5"
+    assert not file.is_file()
+
+    dh1 = get_datahandler(tmpdir1, 'hdf5')
+    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
+    assert file.is_file()
+
+    assert not ohlcv[ohlcv['date'] < startdt].empty
+
+    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
+
+    # Call private function to ensure timerange is filtered in hdf5
+    ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type)
+    ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
+    assert len(ohlcv) == len(ohlcv1)
+    assert ohlcv.equals(ohlcv1)
+    assert ohlcv[ohlcv['date'] < startdt].empty
+    assert ohlcv[ohlcv['date'] > enddt].empty
+
+    # Try loading inexisting file
+    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
+    assert ohlcv.empty
+
+
+@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
+    # Data goes from 2018-01-10 - 2018-01-30
+    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
+    # Mark data goes from to 2021-11-15 2021-11-19
+    ('UNITTEST/USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
+])
+@pytest.mark.parametrize('datahandler', ['hdf5', 'feather', 'parquet'])
+def test_generic_datahandler_ohlcv_load_and_resave(
+    datahandler,
+    testdatadir,
+    tmpdir,
+    pair,
+    timeframe,
+    candle_type,
+    candle_append,
+    startdt, enddt
+):
+    tmpdir1 = Path(tmpdir)
+    tmpdir2 = tmpdir1
+    if candle_type not in ('', 'spot'):
+        tmpdir2 = tmpdir1 / 'futures'
+        tmpdir2.mkdir()
+    # Load data from one common file
+    dhbase = get_datahandler(testdatadir, 'json')
+    ohlcv = dhbase._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
+    assert isinstance(ohlcv, DataFrame)
+    assert len(ohlcv) > 0
+
+    # Get data to test
+    dh = get_datahandler(testdatadir, datahandler)
+
+    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.{dh._get_file_extension()}"
+    assert not file.is_file()
+
+    dh1 = get_datahandler(tmpdir1, datahandler)
+    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
+    assert file.is_file()
+
+    assert not ohlcv[ohlcv['date'] < startdt].empty
+
+    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
+
+    ohlcv = dhbase.ohlcv_load(pair, timeframe, timerange=timerange, candle_type=candle_type)
+    if datahandler == 'hdf5':
+        ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
+        if candle_type == 'mark':
+            ohlcv1['volume'] = 0.0
+    else:
+        ohlcv1 = dh1.ohlcv_load('UNITTEST/NEW', timeframe,
+                                timerange=timerange, candle_type=candle_type)
+
+    assert len(ohlcv) == len(ohlcv1)
+    assert ohlcv.equals(ohlcv1)
+    assert ohlcv[ohlcv['date'] < startdt].empty
+    assert ohlcv[ohlcv['date'] > enddt].empty
+
+    # Try loading inexisting file
+    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
+    assert ohlcv.empty
+
+
+def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 2
+
+
+def test_gethandlerclass():
+    cl = get_datahandlerclass('json')
+    assert cl == JsonDataHandler
+    assert issubclass(cl, IDataHandler)
+
+    cl = get_datahandlerclass('jsongz')
+    assert cl == JsonGzDataHandler
+    assert issubclass(cl, IDataHandler)
+    assert issubclass(cl, JsonDataHandler)
+
+    cl = get_datahandlerclass('hdf5')
+    assert cl == HDF5DataHandler
+    assert issubclass(cl, IDataHandler)
+
+    cl = get_datahandlerclass('feather')
+    assert cl == FeatherDataHandler
+    assert issubclass(cl, IDataHandler)
+
+    cl = get_datahandlerclass('parquet')
+    assert cl == ParquetDataHandler
+    assert issubclass(cl, IDataHandler)
+
+    with pytest.raises(ValueError, match=r"No datahandler for .*"):
+        get_datahandlerclass('DeadBeef')
+
+
+def test_get_datahandler(testdatadir):
+    dh = get_datahandler(testdatadir, 'json')
+    assert type(dh) == JsonDataHandler
+    dh = get_datahandler(testdatadir, 'jsongz')
+    assert type(dh) == JsonGzDataHandler
+    dh1 = get_datahandler(testdatadir, 'jsongz', dh)
+    assert id(dh1) == id(dh)
+
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert type(dh) == HDF5DataHandler

tests/data/test_dataprovider.py

@@ -144,6 +144,77 @@ def test_available_pairs(mocker, default_conf, ohlcv_history):
     assert dp.available_pairs == [("XRP/BTC", timeframe), ("UNITTEST/BTC", timeframe), ]
 
 
+def test_producer_pairs(mocker, default_conf, ohlcv_history):
+    dataprovider = DataProvider(default_conf, None)
+
+    producer = "default"
+    whitelist = ["XRP/BTC", "ETH/BTC"]
+    assert len(dataprovider.get_producer_pairs(producer)) == 0
+
+    dataprovider._set_producer_pairs(whitelist, producer)
+    assert len(dataprovider.get_producer_pairs(producer)) == 2
+
+    new_whitelist = ["BTC/USDT"]
+    dataprovider._set_producer_pairs(new_whitelist, producer)
+    assert dataprovider.get_producer_pairs(producer) == new_whitelist
+
+    assert dataprovider.get_producer_pairs("bad") == []
+
+
+def test_get_producer_df(mocker, default_conf, ohlcv_history):
+    dataprovider = DataProvider(default_conf, None)
+
+    pair = 'BTC/USDT'
+    timeframe = default_conf['timeframe']
+    candle_type = CandleType.SPOT
+
+    empty_la = datetime.fromtimestamp(0, tz=timezone.utc)
+    now = datetime.now(timezone.utc)
+
+    # no data has been added, any request should return an empty dataframe
+    dataframe, la = dataprovider.get_producer_df(pair, timeframe, candle_type)
+    assert dataframe.empty
+    assert la == empty_la
+
+    # the data is added, should return that added dataframe
+    dataprovider._add_external_df(pair, ohlcv_history, now, timeframe, candle_type)
+    dataframe, la = dataprovider.get_producer_df(pair, timeframe, candle_type)
+    assert len(dataframe) > 0
+    assert la > empty_la
+
+    # no data on this producer, should return empty dataframe
+    dataframe, la = dataprovider.get_producer_df(pair, producer_name='bad')
+    assert dataframe.empty
+    assert la == empty_la
+
+    # non existent timeframe, empty dataframe
+    datframe, la = dataprovider.get_producer_df(pair, timeframe='1h')
+    assert dataframe.empty
+    assert la == empty_la
+
+
+def test_emit_df(mocker, default_conf, ohlcv_history):
+    mocker.patch('freqtrade.rpc.rpc_manager.RPCManager.__init__', MagicMock())
+    rpc_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager', MagicMock())
+    send_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager.send_msg', MagicMock())
+
+    dataprovider = DataProvider(default_conf, exchange=None, rpc=rpc_mock)
+    dataprovider_no_rpc = DataProvider(default_conf, exchange=None)
+
+    pair = "BTC/USDT"
+
+    # No emit yet
+    assert send_mock.call_count == 0
+
+    # Rpc is added, we call emit, should call send_msg
+    dataprovider._emit_df(pair, ohlcv_history)
+    assert send_mock.call_count == 1
+
+    # No rpc added, emit called, should not call send_msg
+    dataprovider_no_rpc._emit_df(pair, ohlcv_history)
+    assert send_mock.call_count == 1
+
+
 def test_refresh(mocker, default_conf, ohlcv_history):
     refresh_mock = MagicMock()
     mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)

tests/data/test_entryexitanalysis.py

@@ -124,8 +124,8 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp
     assert '0' in captured.out
     assert '0.01616' in captured.out
     assert '34.049' in captured.out
-    assert '0.104104' in captured.out
-    assert '47.0996' in captured.out
+    assert '0.104411' in captured.out
+    assert '52.8292' in captured.out
 
     # test group 1
     args = get_args(base_args + ['--analysis-groups', "1"])

tests/data/test_history.py

@@ -1,7 +1,6 @@
 # pragma pylint: disable=missing-docstring, protected-access, C0103
 
 import json
-import re
 import uuid
 from pathlib import Path
 from shutil import copyfile
@@ -13,18 +12,17 @@ from pandas import DataFrame
 from pandas.testing import assert_frame_equal
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import AVAILABLE_DATAHANDLERS, DATETIME_PRINT_FORMAT
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data.converter import ohlcv_to_dataframe
-from freqtrade.data.history.hdf5datahandler import HDF5DataHandler
 from freqtrade.data.history.history_utils import (_download_pair_history, _download_trades_history,
                                                   _load_cached_data_for_updating,
                                                   convert_trades_to_ohlcv, get_timerange, load_data,
                                                   load_pair_history, refresh_backtest_ohlcv_data,
                                                   refresh_backtest_trades_data, refresh_data,
                                                   validate_backtest_data)
-from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass
+from freqtrade.data.history.idatahandler import get_datahandler
 from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
-from freqtrade.enums import CandleType, TradingMode
+from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.resolvers import StrategyResolver
@@ -32,25 +30,6 @@ from tests.conftest import (CURRENT_TEST_STRATEGY, get_patched_exchange, log_has
                             patch_exchange)
 
 
-# Change this if modifying UNITTEST/BTC testdatafile
-_BTC_UNITTEST_LENGTH = 13681
-
-
-def _backup_file(file: Path, copy_file: bool = False) -> None:
-    """
-    Backup existing file to avoid deleting the user file
-    :param file: complete path to the file
-    :param copy_file: keep file in place too.
-    :return: None
-    """
-    file_swp = str(file) + '.swp'
-    if file.is_file():
-        file.rename(file_swp)
-
-        if copy_file:
-            copyfile(file_swp, file)
-
-
 def _clean_test_file(file: Path) -> None:
     """
     Backup existing file to avoid deleting the user file
@@ -67,7 +46,7 @@ def _clean_test_file(file: Path) -> None:
         file_swp.rename(file)
 
 
-def test_load_data_30min_timeframe(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_30min_timeframe(caplog, testdatadir) -> None:
     ld = load_pair_history(pair='UNITTEST/BTC', timeframe='30m', datadir=testdatadir)
     assert isinstance(ld, DataFrame)
     assert not log_has(
@@ -76,7 +55,7 @@ def test_load_data_30min_timeframe(mocker, caplog, default_conf, testdatadir) ->
     )
 
 
-def test_load_data_7min_timeframe(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_7min_timeframe(caplog, testdatadir) -> None:
     ld = load_pair_history(pair='UNITTEST/BTC', timeframe='7m', datadir=testdatadir)
     assert isinstance(ld, DataFrame)
     assert ld.empty
@@ -108,7 +87,7 @@ def test_load_data_mark(ohlcv_history, mocker, caplog, testdatadir) -> None:
     )
 
 
-def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_startup_candles(mocker, testdatadir) -> None:
     ltfmock = mocker.patch(
         'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load',
         MagicMock(return_value=DataFrame()))
@@ -398,14 +377,14 @@ def test_load_partial_missing(testdatadir, caplog) -> None:
     td = ((end - start).total_seconds() // 60 // 5) + 1
     assert td != len(data['UNITTEST/BTC'])
 
-    # Shift endtime with +5 - as last candle is dropped (partial candle)
-    end_real = arrow.get(data['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5)
+    # Shift endtime with +5
+    end_real = arrow.get(data['UNITTEST/BTC'].iloc[-1, 0])
     assert log_has(f'UNITTEST/BTC, spot, 5m, '
                    f'data ends at {end_real.strftime(DATETIME_PRINT_FORMAT)}',
                    caplog)
 
 
-def test_init(default_conf, mocker) -> None:
+def test_init(default_conf) -> None:
     assert {} == load_data(
         datadir=Path(''),
         pairs=[],
@@ -468,7 +447,7 @@ def test_get_timerange(default_conf, mocker, testdatadir) -> None:
     )
     min_date, max_date = get_timerange(data)
     assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
-    assert max_date.isoformat() == '2017-11-14T22:58:00+00:00'
+    assert max_date.isoformat() == '2017-11-14T22:59:00+00:00'
 
 
 def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) -> None:
@@ -491,7 +470,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir)
                                   min_date, max_date, timeframe_to_minutes('1m'))
     assert len(caplog.record_tuples) == 1
     assert log_has(
-        "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values",
+        "UNITTEST/BTC has missing frames: expected 14397, got 13681, that's 716 missing values",
         caplog)
 
 
@@ -685,340 +664,3 @@ def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog):
     convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
                             datadir=tmpdir1, timerange=tr, erase=True)
     assert log_has('Could not convert NoDatapair to OHLCV.', caplog)
-
-
-def test_datahandler_ohlcv_get_pairs(testdatadir):
-    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC',
-                          'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC',
-                          'DASH/BTC', 'XRP/ETH'}
-
-    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m', candle_type=CandleType.SPOT)
-    assert set(pairs) == {'UNITTEST/BTC'}
-
-    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
-    assert set(pairs) == {'UNITTEST/BTC'}
-
-    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'}
-
-    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES)
-    assert set(pairs) == {'XRP/USDT'}
-
-    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT:USDT'}
-
-
-@pytest.mark.parametrize('filename,pair,timeframe,candletype', [
-    ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''),
-    ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''),
-    ('BTC-PERP-1h.h5', 'BTC-PERP', '1h', ''),
-    ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''),
-    ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'),
-    ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'),
-    ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'),
-    ('BTC-PERP-1h-index.h5', 'BTC-PERP', '1h', 'index'),
-    ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'),
-])
-def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype):
-    regex = JsonDataHandler._OHLCV_REGEX
-
-    match = re.search(regex, filename)
-    assert len(match.groups()) > 1
-    assert match[1] == pair
-    assert match[2] == timeframe
-    assert match[3] == candletype
-
-
-@pytest.mark.parametrize('input,expected', [
-    ('XMR_USDT', 'XMR/USDT'),
-    ('BTC_USDT', 'BTC/USDT'),
-    ('USDT_BUSD', 'USDT/BUSD'),
-    ('BTC_USDT_USDT', 'BTC/USDT:USDT'),  # Futures
-    ('XRP_USDT_USDT', 'XRP/USDT:USDT'),  # futures
-    ('BTC-PERP', 'BTC-PERP'),
-    ('BTC-PERP_USDT', 'BTC-PERP:USDT'),  # potential FTX case
-    ('UNITTEST_USDT', 'UNITTEST/USDT'),
-])
-def test_rebuild_pair_from_filename(input, expected):
-
-    assert IDataHandler.rebuild_pair_from_filename(input) == expected
-
-
-def test_datahandler_ohlcv_get_available_data(testdatadir):
-    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    # Convert to set to avoid failures due to sorting
-    assert set(paircombs) == {
-        ('UNITTEST/BTC', '5m', CandleType.SPOT),
-        ('ETH/BTC', '5m', CandleType.SPOT),
-        ('XLM/BTC', '5m', CandleType.SPOT),
-        ('TRX/BTC', '5m', CandleType.SPOT),
-        ('LTC/BTC', '5m', CandleType.SPOT),
-        ('XMR/BTC', '5m', CandleType.SPOT),
-        ('ZEC/BTC', '5m', CandleType.SPOT),
-        ('UNITTEST/BTC', '1m', CandleType.SPOT),
-        ('ADA/BTC', '5m', CandleType.SPOT),
-        ('ETC/BTC', '5m', CandleType.SPOT),
-        ('NXT/BTC', '5m', CandleType.SPOT),
-        ('DASH/BTC', '5m', CandleType.SPOT),
-        ('XRP/ETH', '1m', CandleType.SPOT),
-        ('XRP/ETH', '5m', CandleType.SPOT),
-        ('UNITTEST/BTC', '30m', CandleType.SPOT),
-        ('UNITTEST/BTC', '8m', CandleType.SPOT),
-        ('NOPAIR/XXX', '4m', CandleType.SPOT),
-    }
-
-    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.FUTURES)
-    # Convert to set to avoid failures due to sorting
-    assert set(paircombs) == {
-        ('UNITTEST/USDT', '1h', 'mark'),
-        ('XRP/USDT', '1h', 'futures'),
-        ('XRP/USDT', '1h', 'mark'),
-        ('XRP/USDT', '8h', 'mark'),
-        ('XRP/USDT', '8h', 'funding_rate'),
-    }
-
-    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    assert set(paircombs) == {('UNITTEST/BTC', '8m', CandleType.SPOT)}
-    paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)}
-
-
-def test_jsondatahandler_trades_get_pairs(testdatadir):
-    pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
-
-
-def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = JsonGzDataHandler(testdatadir)
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 2
-
-
-def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
-    dh = JsonDataHandler(testdatadir)
-    df = dh.ohlcv_load('XRP/ETH', '5m', 'spot')
-    assert len(df) == 711
-
-    df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark")
-    assert len(df_mark) == 99
-
-    df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot')
-    assert len(df_no_mark) == 0
-
-    # Failure case (empty array)
-    df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', 'spot')
-    assert len(df1) == 0
-    assert log_has("Could not load data for NOPAIR/XXX.", caplog)
-    assert df.columns.equals(df1.columns)
-
-
-def test_jsondatahandler_trades_load(testdatadir, caplog):
-    dh = JsonGzDataHandler(testdatadir)
-    logmsg = "Old trades format detected - converting"
-    dh.trades_load('XRP/ETH')
-    assert not log_has(logmsg, caplog)
-
-    # Test conversation is happening
-    dh.trades_load('XRP/OLD')
-    assert log_has(logmsg, caplog)
-
-
-def test_jsondatahandler_trades_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = JsonGzDataHandler(testdatadir)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 1
-
-
-@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-def test_datahandler_ohlcv_append(datahandler, testdatadir, ):
-    dh = get_datahandler(testdatadir, datahandler)
-    with pytest.raises(NotImplementedError):
-        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.SPOT)
-    with pytest.raises(NotImplementedError):
-        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.MARK)
-
-
-@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-def test_datahandler_trades_append(datahandler, testdatadir):
-    dh = get_datahandler(testdatadir, datahandler)
-    with pytest.raises(NotImplementedError):
-        dh.trades_append('UNITTEST/ETH', [])
-
-
-def test_hdf5datahandler_trades_get_pairs(testdatadir):
-    pairs = HDF5DataHandler.trades_get_pairs(testdatadir)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH'}
-
-
-def test_hdf5datahandler_trades_load(testdatadir):
-    dh = HDF5DataHandler(testdatadir)
-    trades = dh.trades_load('XRP/ETH')
-    assert isinstance(trades, list)
-
-    trades1 = dh.trades_load('UNITTEST/NONEXIST')
-    assert trades1 == []
-    # data goes from 2019-10-11 - 2019-10-13
-    timerange = TimeRange.parse_timerange('20191011-20191012')
-
-    trades2 = dh._trades_load('XRP/ETH', timerange)
-    assert len(trades) > len(trades2)
-    # Check that ID is None (If it's nan, it's wrong)
-    assert trades2[0][2] is None
-
-    # unfiltered load has trades before starttime
-    assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0
-    # filtered list does not have trades before starttime
-    assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0
-    # unfiltered load has trades after endtime
-    assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0
-    # filtered list does not have trades after endtime
-    assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0
-
-
-def test_hdf5datahandler_trades_store(testdatadir, tmpdir):
-    tmpdir1 = Path(tmpdir)
-    dh = HDF5DataHandler(testdatadir)
-    trades = dh.trades_load('XRP/ETH')
-
-    dh1 = HDF5DataHandler(tmpdir1)
-    dh1.trades_store('XRP/NEW', trades)
-    file = tmpdir1 / 'XRP_NEW-trades.h5'
-    assert file.is_file()
-    # Load trades back
-    trades_new = dh1.trades_load('XRP/NEW')
-
-    assert len(trades_new) == len(trades)
-    assert trades[0][0] == trades_new[0][0]
-    assert trades[0][1] == trades_new[0][1]
-    # assert trades[0][2] == trades_new[0][2]  # This is nan - so comparison does not make sense
-    assert trades[0][3] == trades_new[0][3]
-    assert trades[0][4] == trades_new[0][4]
-    assert trades[0][5] == trades_new[0][5]
-    assert trades[0][6] == trades_new[0][6]
-    assert trades[-1][0] == trades_new[-1][0]
-    assert trades[-1][1] == trades_new[-1][1]
-    # assert trades[-1][2] == trades_new[-1][2]  # This is nan - so comparison does not make sense
-    assert trades[-1][3] == trades_new[-1][3]
-    assert trades[-1][4] == trades_new[-1][4]
-    assert trades[-1][5] == trades_new[-1][5]
-    assert trades[-1][6] == trades_new[-1][6]
-
-
-def test_hdf5datahandler_trades_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = HDF5DataHandler(testdatadir)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 1
-
-
-@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
-    # Data goes from 2018-01-10 - 2018-01-30
-    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
-    # Mark data goes from to 2021-11-15 2021-11-19
-    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
-])
-def test_hdf5datahandler_ohlcv_load_and_resave(
-    testdatadir,
-    tmpdir,
-    pair,
-    timeframe,
-    candle_type,
-    candle_append,
-    startdt, enddt
-):
-    tmpdir1 = Path(tmpdir)
-    tmpdir2 = tmpdir1
-    if candle_type not in ('', 'spot'):
-        tmpdir2 = tmpdir1 / 'futures'
-        tmpdir2.mkdir()
-    dh = HDF5DataHandler(testdatadir)
-    ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
-    assert isinstance(ohlcv, DataFrame)
-    assert len(ohlcv) > 0
-
-    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5"
-    assert not file.is_file()
-
-    dh1 = HDF5DataHandler(tmpdir1)
-    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
-    assert file.is_file()
-
-    assert not ohlcv[ohlcv['date'] < startdt].empty
-
-    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
-
-    # Call private function to ensure timerange is filtered in hdf5
-    ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type)
-    ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
-    assert len(ohlcv) == len(ohlcv1)
-    assert ohlcv.equals(ohlcv1)
-    assert ohlcv[ohlcv['date'] < startdt].empty
-    assert ohlcv[ohlcv['date'] > enddt].empty
-
-    # Try loading inexisting file
-    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
-    assert ohlcv.empty
-
-
-def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = HDF5DataHandler(testdatadir)
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 2
-
-
-def test_gethandlerclass():
-    cl = get_datahandlerclass('json')
-    assert cl == JsonDataHandler
-    assert issubclass(cl, IDataHandler)
-    cl = get_datahandlerclass('jsongz')
-    assert cl == JsonGzDataHandler
-    assert issubclass(cl, IDataHandler)
-    assert issubclass(cl, JsonDataHandler)
-    cl = get_datahandlerclass('hdf5')
-    assert cl == HDF5DataHandler
-    assert issubclass(cl, IDataHandler)
-    with pytest.raises(ValueError, match=r"No datahandler for .*"):
-        get_datahandlerclass('DeadBeef')
-
-
-def test_get_datahandler(testdatadir):
-    dh = get_datahandler(testdatadir, 'json')
-    assert type(dh) == JsonDataHandler
-    dh = get_datahandler(testdatadir, 'jsongz')
-    assert type(dh) == JsonGzDataHandler
-    dh1 = get_datahandler(testdatadir, 'jsongz', dh)
-    assert id(dh1) == id(dh)
-
-    dh = get_datahandler(testdatadir, 'hdf5')
-    assert type(dh) == HDF5DataHandler

tests/exchange/test_binance.py

@@ -501,6 +501,24 @@ def test_fill_leverage_tiers_binance_dryrun(default_conf, mocker, leverage_tiers
         assert len(v) == len(value)
 
 
+def test_additional_exchange_init_binance(default_conf, mocker):
+    api_mock = MagicMock()
+    api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": True})
+    api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": True})
+    default_conf['dry_run'] = False
+    default_conf['trading_mode'] = TradingMode.FUTURES
+    default_conf['margin_mode'] = MarginMode.ISOLATED
+    with pytest.raises(OperationalException,
+                       match=r"Hedge Mode is not supported.*\nMulti-Asset Mode is not supported.*"):
+        get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock)
+    api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": False})
+    api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": False})
+    exchange = get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock)
+    assert exchange
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'binance',
+                           "additional_exchange_init", "fapiPrivateGetPositionsideDual")
+
+
 def test__set_leverage_binance(mocker, default_conf):
 
     api_mock = MagicMock()

tests/exchange/test_ccxt_compat.py

@@ -137,6 +137,7 @@ def exchange_futures(request, exchange_conf, class_mocker):
             'freqtrade.exchange.binance.Binance.fill_leverage_tiers')
         class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees')
         class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init')
+        class_mocker.patch('freqtrade.exchange.binance.Binance.additional_exchange_init')
         class_mocker.patch('freqtrade.exchange.exchange.Exchange.load_cached_leverage_tiers',
                            return_value=None)
         class_mocker.patch('freqtrade.exchange.exchange.Exchange.cache_leverage_tiers')

tests/exchange/test_exchange.py

@@ -20,6 +20,7 @@ from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_pr
                                 timeframe_to_prev_date, timeframe_to_seconds)
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT,
                                        calculate_backoff, remove_credentials)
+from freqtrade.exchange.exchange import amount_to_contract_precision
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
 from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re
 
@@ -4470,6 +4471,7 @@ def test__amount_to_contracts(
     ('ADA/USDT:USDT', 10.4445555, 10.4, 10.444),
     ('LTC/ETH', 30, 30, 30),
     ('LTC/USD', 30, 30, 30),
+    ('ADA/USDT:USDT', 1.17, 1.1, 1.17),
     # contract size of 10
     ('ETH/USDT:USDT', 10.111, 10.1, 10),
     ('ETH/USDT:USDT', 10.188, 10.1, 10),
@@ -4497,6 +4499,20 @@ def test_amount_to_contract_precision(
     assert result_size == expected_fut
 
 
+@pytest.mark.parametrize('amount,precision,precision_mode,contract_size,expected', [
+    (1.17, 1.0, 4, 0.01, 1.17),  # Tick size
+    (1.17, 1.0, 2, 0.01, 1.17),  #
+    (1.16, 1.0, 4, 0.01, 1.16),  #
+    (1.16, 1.0, 2, 0.01, 1.16),  #
+    (1.13, 1.0, 2, 0.01, 1.13),  #
+    (10.988, 1.0, 2, 10, 10),
+    (10.988, 1.0, 4, 10, 10),
+])
+def test_amount_to_contract_precision2(amount, precision, precision_mode, contract_size, expected):
+    res = amount_to_contract_precision(amount, precision, precision_mode, contract_size)
+    assert pytest.approx(res) == expected
+
+
 @pytest.mark.parametrize('exchange_name,open_rate,is_short,trading_mode,margin_mode', [
     # Bittrex
     ('bittrex', 2.0, False, 'spot', None),

tests/freqai/test_freqai_backtesting.py

@@ -3,21 +3,21 @@ from datetime import datetime, timezone
 from pathlib import Path
 from unittest.mock import PropertyMock
 
-import pytest
-
-from freqtrade.commands.optimize_commands import start_backtesting
-from freqtrade.exceptions import OperationalException
+from freqtrade.commands.optimize_commands import setup_optimize_configuration
+from freqtrade.enums import RunMode
 from freqtrade.optimize.backtesting import Backtesting
 from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
 
-def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir):
+def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, caplog):
     patch_exchange(mocker)
 
+    now = datetime.now(timezone.utc)
     mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
                  PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
-    # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
+    mocker.patch('freqtrade.optimize.backtesting.history.load_data')
+    mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
 
     patched_configuration_load_config_file(mocker, freqai_conf)
 
@@ -30,9 +30,11 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir):
         '--strategy-list', CURRENT_TEST_STRATEGY
     ]
     args = get_args(args)
-    with pytest.raises(OperationalException,
-                       match=r"You can't use strategy_list and freqai at the same time\."):
-        start_backtesting(args)
+    bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
+    Backtesting(bt_config)
+    assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical '
+                      'populate_any_indicators.', caplog)
+    Backtesting.cleanup()
 
 
 def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):

tests/freqai/test_freqai_datakitchen.py

@@ -71,10 +71,7 @@ def test_use_DBSCAN_to_remove_outliers(mocker, freqai_conf, caplog):
     freqai = make_data_dictionary(mocker, freqai_conf)
     # freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 1})
     freqai.dk.use_DBSCAN_to_remove_outliers(predict=False)
-    assert log_has_re(
-        "DBSCAN found eps of 1.75.",
-        caplog,
-    )
+    assert log_has_re(r"DBSCAN found eps of 2\.3\d\.", caplog)
 
 
 def test_compute_distances(mocker, freqai_conf):
@@ -89,7 +86,7 @@ def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf,
     freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 0.1})
     freqai.dk.use_SVM_to_remove_outliers(predict=False)
     assert log_has_re(
-        "SVM detected 7.36%",
+        "SVM detected 8.66%",
         caplog,
     )
 

tests/optimize/conftest.py

@@ -6,6 +6,7 @@ import pandas as pd
 import pytest
 
 from freqtrade.enums import ExitType, RunMode
+from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.optimize.hyperopt import Hyperopt
 from tests.conftest import patch_exchange
 
@@ -28,6 +29,13 @@ def hyperopt_conf(default_conf):
     return hyperconf
 
 
+@pytest.fixture(autouse=True)
+def backtesting_cleanup() -> None:
+    yield None
+
+    Backtesting.cleanup()
+
+
 @pytest.fixture(scope='function')
 def hyperopt(hyperopt_conf, mocker):
 

tests/optimize/test_backtesting.py

@@ -52,13 +52,6 @@ def trim_dictlist(dict_list, num):
     return new
 
 
-@pytest.fixture(autouse=True)
-def backtesting_cleanup() -> None:
-    yield None
-
-    Backtesting.cleanup()
-
-
 def load_data_test(what, testdatadir):
     timerange = TimeRange.parse_timerange('1510694220-1510700340')
     data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir,
@@ -87,7 +80,7 @@ def load_data_test(what, testdatadir):
         data.loc[:, 'close'] = np.sin(data.index * hz) / 1000 + base
 
     return {'UNITTEST/BTC': clean_ohlcv_dataframe(data, timeframe='1m', pair='UNITTEST/BTC',
-                                                  fill_missing=True)}
+                                                  fill_missing=True, drop_incomplete=True)}
 
 
 # FIX: fixturize this?
@@ -330,7 +323,7 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
     backtesting = Backtesting(default_conf)
     backtesting._set_strategy(backtesting.strategylist[0])
     processed = backtesting.strategy.advise_all_indicators(data)
-    assert len(processed['UNITTEST/BTC']) == 102
+    assert len(processed['UNITTEST/BTC']) == 103
 
     # Load strategy to compare the result between Backtesting function and strategy are the same
     strategy = StrategyResolver.load_strategy(default_conf)
@@ -434,7 +427,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
 
     default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}]
     with pytest.raises(OperationalException,
-                       match=r'VolumePairList not allowed for backtesting\..*StaticPairlist.*'):
+                       match=r'VolumePairList not allowed for backtesting\..*StaticPairList.*'):
         Backtesting(default_conf)
 
     default_conf.update({
@@ -467,7 +460,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti
 
     default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}]
     with pytest.raises(OperationalException,
-                       match=r'VolumePairList not allowed for backtesting\..*StaticPairlist.*'):
+                       match=r'VolumePairList not allowed for backtesting\..*StaticPairList.*'):
         Backtesting(default_conf)
 
     default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}]
@@ -846,7 +839,7 @@ def test_backtest_trim_no_data_left(default_conf, fee, mocker, testdatadir) -> N
     data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'],
                              timerange=timerange)
     df = data['UNITTEST/BTC']
-    df.loc[:, 'date'] = df.loc[:, 'date'] - timedelta(days=1)
+    df['date'] = df.loc[:, 'date'] - timedelta(days=1)
     # Trimming 100 candles, so after 2nd trimming, no candle is left.
     df = df.iloc[:100]
     data['XRP/USDT'] = df
@@ -1172,9 +1165,9 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
+        'up to 2017-11-14 22:59:00 (0 days).',
         'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
+        'up to 2017-11-14 22:59:00 (0 days).',
         'Parameter --enable-position-stacking detected ...'
     ]
 
@@ -1251,9 +1244,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
+        'up to 2017-11-14 22:59:00 (0 days).',
         'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
+        'up to 2017-11-14 22:59:00 (0 days).',
         'Parameter --enable-position-stacking detected ...',
         f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}',
         'Running backtesting for Strategy StrategyTestV2',
@@ -1362,9 +1355,9 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
+        'up to 2017-11-14 22:59:00 (0 days).',
         'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
+        'up to 2017-11-14 22:59:00 (0 days).',
         'Parameter --enable-position-stacking detected ...',
         f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}',
         'Running backtesting for Strategy StrategyTestV2',
@@ -1378,7 +1371,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
     assert 'EXIT REASON STATS' in captured.out
     assert 'DAY BREAKDOWN' in captured.out
     assert 'LEFT OPEN TRADES REPORT' in captured.out
-    assert '2017-11-14 21:17:00 -> 2017-11-14 22:58:00 | Max open trades : 1' in captured.out
+    assert '2017-11-14 21:17:00 -> 2017-11-14 22:59:00 | Max open trades : 1' in captured.out
     assert 'STRATEGY SUMMARY' in captured.out
 
 
@@ -1510,9 +1503,9 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker,
         'Parameter -i/--timeframe detected ... Using timeframe: 1h ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2021-11-17 01:00:00 '
-        'up to 2021-11-21 03:00:00 (4 days).',
+        'up to 2021-11-21 04:00:00 (4 days).',
         'Backtesting with data from 2021-11-17 21:00:00 '
-        'up to 2021-11-21 03:00:00 (3 days).',
+        'up to 2021-11-21 04:00:00 (3 days).',
         'XRP/USDT, funding_rate, 8h, data starts at 2021-11-18 00:00:00',
         'XRP/USDT, mark, 8h, data starts at 2021-11-18 00:00:00',
         f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}',
@@ -1623,9 +1616,9 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker,
         'Parameter --timeframe-detail detected, using 1m for intra-candle backtesting ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2019-10-11 00:00:00 '
-        'up to 2019-10-13 11:10:00 (2 days).',
+        'up to 2019-10-13 11:15:00 (2 days).',
         'Backtesting with data from 2019-10-11 01:40:00 '
-        'up to 2019-10-13 11:10:00 (2 days).',
+        'up to 2019-10-13 11:15:00 (2 days).',
         f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}',
     ]
 
@@ -1726,7 +1719,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
+        'up to 2017-11-14 22:59:00 (0 days).',
         'Parameter --enable-position-stacking detected ...',
     ]
 
@@ -1739,7 +1732,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
             'Running backtesting for Strategy StrategyTestV2',
             'Running backtesting for Strategy StrategyTestV3',
             'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
-            'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
+            'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:59:00 (0 days).',
         ]
     elif run_id == '2' and min_backtest_date < start_time:
         assert backtestmock.call_count == 0
@@ -1752,7 +1745,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
             'Reusing result of previous backtest for StrategyTestV2',
             'Running backtesting for Strategy StrategyTestV3',
             'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
-            'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
+            'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:59:00 (0 days).',
         ]
         assert backtestmock.call_count == 1
 

tests/optimize/test_backtesting_adjust_position.py

@@ -93,11 +93,16 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
                 t["close_rate"], 6) < round(ln.iloc[0]["high"], 6))
 
 
-def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> None:
+@pytest.mark.parametrize('leverage', [
+    1, 2
+])
+def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, leverage) -> None:
     default_conf['use_exit_signal'] = False
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
     mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=10)
     mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf'))
+    mocker.patch("freqtrade.exchange.Exchange.get_max_leverage", return_value=10)
+
     patch_exchange(mocker)
     default_conf.update({
         "stake_amount": 100.0,
@@ -105,6 +110,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> Non
         "strategy": "StrategyTestV3"
     })
     backtesting = Backtesting(default_conf)
+    backtesting._can_short = True
     backtesting._set_strategy(backtesting.strategylist[0])
     pair = 'XRP/USDT'
     row = [
@@ -120,18 +126,19 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> Non
             '',  # enter_tag
             '',  # exit_tag
             ]
+    backtesting.strategy.leverage = MagicMock(return_value=leverage)
     trade = backtesting._enter_trade(pair, row=row, direction='long')
     trade.orders[0].close_bt_order(row[0], trade)
     assert trade
     assert pytest.approx(trade.stake_amount) == 100.0
-    assert pytest.approx(trade.amount) == 47.61904762
+    assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 1
     backtesting.strategy.adjust_trade_position = MagicMock(return_value=None)
 
     trade = backtesting._get_adjust_trade_entry_for_candle(trade, row)
     assert trade
     assert pytest.approx(trade.stake_amount) == 100.0
-    assert pytest.approx(trade.amount) == 47.61904762
+    assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 1
     # Increase position by 100
     backtesting.strategy.adjust_trade_position = MagicMock(return_value=100)
@@ -140,7 +147,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> Non
 
     assert trade
     assert pytest.approx(trade.stake_amount) == 200.0
-    assert pytest.approx(trade.amount) == 95.23809524
+    assert pytest.approx(trade.amount) == 95.23809524 * leverage
     assert len(trade.orders) == 2
 
     # Reduce by more than amount - no change to trade.
@@ -150,7 +157,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> Non
 
     assert trade
     assert pytest.approx(trade.stake_amount) == 200.0
-    assert pytest.approx(trade.amount) == 95.23809524
+    assert pytest.approx(trade.amount) == 95.23809524 * leverage
     assert len(trade.orders) == 2
     assert trade.nr_of_successful_entries == 2
 
@@ -160,7 +167,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> Non
 
     assert trade
     assert pytest.approx(trade.stake_amount) == 100.0
-    assert pytest.approx(trade.amount) == 47.61904762
+    assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 3
     assert trade.nr_of_successful_entries == 2
     assert trade.nr_of_successful_exits == 1
@@ -171,7 +178,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> Non
 
     assert trade
     assert pytest.approx(trade.stake_amount) == 100.0
-    assert pytest.approx(trade.amount) == 47.61904762
+    assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 3
     assert trade.nr_of_successful_entries == 2
     assert trade.nr_of_successful_exits == 1

tests/optimize/test_optimize_reports.py

@@ -1,6 +1,7 @@
 import re
 from datetime import timedelta
 from pathlib import Path
+from shutil import copyfile
 
 import joblib
 import pandas as pd
@@ -25,7 +26,22 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene
                                                  text_table_exit_reason, text_table_strategy)
 from freqtrade.resolvers.strategy_resolver import StrategyResolver
 from tests.conftest import CURRENT_TEST_STRATEGY
-from tests.data.test_history import _backup_file, _clean_test_file
+from tests.data.test_history import _clean_test_file
+
+
+def _backup_file(file: Path, copy_file: bool = False) -> None:
+    """
+    Backup existing file to avoid deleting the user file
+    :param file: complete path to the file
+    :param copy_file: keep file in place too.
+    :return: None
+    """
+    file_swp = str(file) + '.swp'
+    if file.is_file():
+        file.rename(file_swp)
+
+        if copy_file:
+            copyfile(file_swp, file)
 
 
 def test_text_table_bt_results():

tests/plugins/test_pairlist.py

@@ -9,6 +9,7 @@ import pytest
 import time_machine
 
 from freqtrade.constants import AVAILABLE_PAIRLISTS
+from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType, RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence import Trade
@@ -40,6 +41,12 @@ def whitelist_conf(default_conf):
             "sort_key": "quoteVolume",
         },
     ]
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [],
+        }
+    })
     return default_conf
 
 
@@ -126,7 +133,7 @@ def test_log_cached(mocker, static_pl_conf, markets, tickers):
 def test_load_pairlist_noexist(mocker, markets, default_conf):
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
-    plm = PairListManager(freqtrade.exchange, default_conf)
+    plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
     with pytest.raises(OperationalException,
                        match=r"Impossible to load Pairlist 'NonexistingPairList'. "
                              r"This class does not exist or contains Python code errors."):
@@ -137,7 +144,7 @@ def test_load_pairlist_noexist(mocker, markets, default_conf):
 def test_load_pairlist_verify_multi(mocker, markets_static, default_conf):
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_static))
-    plm = PairListManager(freqtrade.exchange, default_conf)
+    plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
     # Call different versions one after the other, should always consider what was passed in
     # and have no side-effects (therefore the same check multiple times)
     assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', ], print) == ['ETH/BTC', 'XRP/BTC']
@@ -269,7 +276,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co
     with pytest.raises(OperationalException,
                        match=r'`number_assets` not specified. Please check your configuration '
                              r'for "pairlist.config.number_assets"'):
-        PairListManager(freqtrade.exchange, whitelist_conf)
+        PairListManager(freqtrade.exchange, whitelist_conf, MagicMock())
 
 
 def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_conf_2):
@@ -622,10 +629,10 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers,
 
     # create candles for high volume with all candles high volume, but very low price.
     ohlcv_history_high_volume = ohlcv_history.copy()
-    ohlcv_history_high_volume.loc[:, 'volume'] = 10
-    ohlcv_history_high_volume.loc[:, 'low'] = ohlcv_history_high_volume.loc[:, 'low'] * 0.01
-    ohlcv_history_high_volume.loc[:, 'high'] = ohlcv_history_high_volume.loc[:, 'high'] * 0.01
-    ohlcv_history_high_volume.loc[:, 'close'] = ohlcv_history_high_volume.loc[:, 'close'] * 0.01
+    ohlcv_history_high_volume['volume'] = 10
+    ohlcv_history_high_volume['low'] = ohlcv_history_high_volume.loc[:, 'low'] * 0.01
+    ohlcv_history_high_volume['high'] = ohlcv_history_high_volume.loc[:, 'high'] * 0.01
+    ohlcv_history_high_volume['close'] = ohlcv_history_high_volume.loc[:, 'close'] * 0.01
 
     mocker.patch('freqtrade.exchange.ftx.Ftx.market_is_tradable', return_value=True)
 
@@ -694,7 +701,7 @@ def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
 
     with pytest.raises(OperationalException,
                        match=r"PrecisionFilter can only work with stoploss defined\..*"):
-        PairListManager(MagicMock, whitelist_conf)
+        PairListManager(MagicMock, whitelist_conf, MagicMock())
 
 
 def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
@@ -703,7 +710,7 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
         del Trade.query
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
     exchange = get_patched_exchange(mocker, whitelist_conf)
-    pm = PairListManager(exchange, whitelist_conf)
+    pm = PairListManager(exchange, whitelist_conf, MagicMock())
     pm.refresh_pairlist()
 
     assert log_has("PerformanceFilter is not available in this mode.", caplog)
@@ -1167,6 +1174,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo
      "[{'OffsetFilter': 'OffsetFilter - Taking 10 Pairs, starting from 5.'}]",
      None
      ),
+    ({"method": "ProducerPairList"},
+     "[{'ProducerPairList': 'ProducerPairList - default'}]",
+     None
+     ),
 ])
 def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
                           desc_expected, exception_expected):
@@ -1341,3 +1352,77 @@ def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected):
             expand_pairlist(wildcardlist, pairs, keep_invalid=True)
     else:
         assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected)
+
+
+def test_ProducerPairlist_no_emc(mocker, whitelist_conf):
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    whitelist_conf['pairlists'] = [
+        {
+            "method": "ProducerPairList",
+            "number_assets": 10,
+            "producer_name": "hello_world",
+        }
+    ]
+    del whitelist_conf['external_message_consumer']
+
+    with pytest.raises(OperationalException,
+                       match=r"ProducerPairList requires external_message_consumer to be enabled."):
+        get_patched_freqtradebot(mocker, whitelist_conf)
+
+
+def test_ProducerPairlist(mocker, whitelist_conf, markets):
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          )
+    whitelist_conf['pairlists'] = [
+        {
+            "method": "ProducerPairList",
+            "number_assets": 2,
+            "producer_name": "hello_world",
+        }
+    ]
+    whitelist_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "hello_world",
+                    "host": "null",
+                    "port": 9891,
+                    "ws_token": "dummy",
+                }
+            ]
+        }
+    })
+
+    exchange = get_patched_exchange(mocker, whitelist_conf)
+    dp = DataProvider(whitelist_conf, exchange, None)
+    pairs = ['ETH/BTC', 'LTC/BTC', 'XRP/BTC']
+    # different producer
+    dp._set_producer_pairs(pairs + ['MEEP/USDT'], 'default')
+    pm = PairListManager(exchange, whitelist_conf, dp)
+    pm.refresh_pairlist()
+    assert pm.whitelist == []
+    # proper producer
+    dp._set_producer_pairs(pairs, 'hello_world')
+    pm.refresh_pairlist()
+
+    # Pairlist reduced to 2
+    assert pm.whitelist == pairs[:2]
+    assert len(pm.whitelist) == 2
+    whitelist_conf['exchange']['pair_whitelist'] = ['TKN/BTC']
+
+    whitelist_conf['pairlists'] = [
+        {"method": "StaticPairList"},
+        {
+            "method": "ProducerPairList",
+            "producer_name": "hello_world",
+        }
+    ]
+    pm = PairListManager(exchange, whitelist_conf, dp)
+    pm.refresh_pairlist()
+    assert len(pm.whitelist) == 4
+    assert pm.whitelist == ['TKN/BTC'] + pairs

tests/rpc/test_rpc.py

@@ -45,7 +45,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
 
     freqtradebot.enter_positions()
     trades = Trade.get_open_trades()
-    trades[0].open_order_id = None
     freqtradebot.exit_positions(trades)
 
     results = rpc._rpc_trade_status()
@@ -1031,6 +1030,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None:
 
 def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open) -> None:
     default_conf['force_entry_enable'] = True
+    default_conf['max_open_trades'] = 0
     mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
     buy_mm = MagicMock(return_value=limit_buy_order_open)
     mocker.patch.multiple(
@@ -1045,6 +1045,10 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open
     patch_get_signal(freqtradebot)
     rpc = RPC(freqtradebot)
     pair = 'ETH/BTC'
+    with pytest.raises(RPCException, match='Maximum number of trades is reached.'):
+        rpc._rpc_force_entry(pair, None)
+    freqtradebot.config['max_open_trades'] = 5
+
     trade = rpc._rpc_force_entry(pair, None)
     assert isinstance(trade, Trade)
     assert trade.pair == pair

tests/rpc/test_rpc_apiserver.py

@@ -3,6 +3,8 @@ Unit test file for rpc/api_server.py
 """
 
 import json
+import logging
+import time
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from unittest.mock import ANY, MagicMock, PropertyMock
@@ -10,7 +12,7 @@ from unittest.mock import ANY, MagicMock, PropertyMock
 import pandas as pd
 import pytest
 import uvicorn
-from fastapi import FastAPI
+from fastapi import FastAPI, WebSocketDisconnect
 from fastapi.exceptions import HTTPException
 from fastapi.testclient import TestClient
 from requests.auth import _basic_auth_str
@@ -31,6 +33,7 @@ from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_mock_
 BASE_URI = "/api/v1"
 _TEST_USER = "FreqTrader"
 _TEST_PASS = "SuperSecurePassword1!"
+_TEST_WS_TOKEN = "secret_Ws_t0ken"
 
 
 @pytest.fixture
@@ -44,17 +47,21 @@ def botclient(default_conf, mocker):
                                         "CORS_origins": ['http://example.com'],
                                         "username": _TEST_USER,
                                         "password": _TEST_PASS,
+                                        "ws_token": _TEST_WS_TOKEN
                                         }})
 
     ftbot = get_patched_freqtradebot(mocker, default_conf)
     rpc = RPC(ftbot)
     mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api', MagicMock())
+    apiserver = None
     try:
         apiserver = ApiServer(default_conf)
         apiserver.add_rpc_handler(rpc)
         yield ftbot, TestClient(apiserver.app)
         # Cleanup ... ?
     finally:
+        if apiserver:
+            apiserver.cleanup()
         ApiServer.shutdown()
 
 
@@ -154,6 +161,25 @@ def test_api_auth():
         get_user_from_token(b'not_a_token', 'secret1234')
 
 
+def test_api_ws_auth(botclient):
+    ftbot, client = botclient
+    def url(token): return f"/api/v1/message/ws?token={token}"
+
+    bad_token = "bad-ws_token"
+    with pytest.raises(WebSocketDisconnect):
+        with client.websocket_connect(url(bad_token)) as websocket:
+            websocket.receive()
+
+    good_token = _TEST_WS_TOKEN
+    with client.websocket_connect(url(good_token)) as websocket:
+        pass
+
+    jwt_secret = ftbot.config['api_server'].get('jwt_secret_key', 'super-secret')
+    jwt_token = create_token({'identity': {'u': 'Freqtrade'}}, jwt_secret)
+    with client.websocket_connect(url(jwt_token)) as websocket:
+        pass
+
+
 def test_api_unauthorized(botclient):
     ftbot, client = botclient
     rc = client.get(f"{BASE_URI}/ping")
@@ -261,6 +287,7 @@ def test_api__init__(default_conf, mocker):
     with pytest.raises(OperationalException, match="RPC Handler already attached."):
         apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf)))
 
+    apiserver.cleanup()
     ApiServer.shutdown()
 
 
@@ -388,6 +415,7 @@ def test_api_run(default_conf, mocker, caplog):
                  MagicMock(side_effect=Exception))
     apiserver.start_api()
     assert log_has("Api server failed to start.", caplog)
+    apiserver.cleanup()
     ApiServer.shutdown()
 
 
@@ -410,6 +438,7 @@ def test_api_cleanup(default_conf, mocker, caplog):
     apiserver.cleanup()
     assert apiserver._server.cleanup.call_count == 1
     assert log_has("Stopping API Server", caplog)
+    assert log_has("Stopping API Server background tasks", caplog)
     ApiServer.shutdown()
 
 
@@ -1449,6 +1478,10 @@ def test_api_strategy(botclient):
     rc = client_get(client, f"{BASE_URI}/strategy/NoStrat")
     assert_response(rc, 404)
 
+    # Disallow base64 strategies
+    rc = client_get(client, f"{BASE_URI}/strategy/xx:cHJpbnQoImhlbGxvIHdvcmxkIik=")
+    assert_response(rc, 500)
+
 
 def test_list_available_pairs(botclient):
     ftbot, client = botclient
@@ -1622,6 +1655,11 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
     assert not result['running']
     assert result['status_msg'] == 'Backtest reset'
 
+    # Disallow base64 strategies
+    data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik="
+    rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data))
+    assert_response(rc, 500)
+
 
 def test_api_backtest_history(botclient, mocker, testdatadir):
     ftbot, client = botclient
@@ -1664,3 +1702,93 @@ def test_health(botclient):
     ret = rc.json()
     assert ret['last_process_ts'] == 0
     assert ret['last_process'] == '1970-01-01T00:00:00+00:00'
+
+
+def test_api_ws_subscribe(botclient, mocker):
+    ftbot, client = botclient
+    ws_url = f"/api/v1/message/ws?token={_TEST_WS_TOKEN}"
+
+    sub_mock = mocker.patch('freqtrade.rpc.api_server.ws.WebSocketChannel.set_subscriptions')
+
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({'type': 'subscribe', 'data': ['whitelist']})
+
+    # Check call count is now 1 as we sent a valid subscribe request
+    assert sub_mock.call_count == 1
+
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({'type': 'subscribe', 'data': 'whitelist'})
+
+    # Call count hasn't changed as the subscribe request was invalid
+    assert sub_mock.call_count == 1
+
+
+def test_api_ws_requests(botclient, mocker, caplog):
+    caplog.set_level(logging.DEBUG)
+
+    ftbot, client = botclient
+    ws_url = f"/api/v1/message/ws?token={_TEST_WS_TOKEN}"
+
+    # Test whitelist request
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({"type": "whitelist", "data": None})
+        response = ws.receive_json()
+
+    assert log_has_re(r"Request of type whitelist from.+", caplog)
+    assert response['type'] == "whitelist"
+
+    # Test analyzed_df request
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({"type": "analyzed_df", "data": {}})
+        response = ws.receive_json()
+
+    assert log_has_re(r"Request of type analyzed_df from.+", caplog)
+    assert response['type'] == "analyzed_df"
+
+    caplog.clear()
+    # Test analyzed_df request with data
+    with client.websocket_connect(ws_url) as ws:
+        ws.send_json({"type": "analyzed_df", "data": {"limit": 100}})
+        response = ws.receive_json()
+
+    assert log_has_re(r"Request of type analyzed_df from.+", caplog)
+    assert response['type'] == "analyzed_df"
+
+
+def test_api_ws_send_msg(default_conf, mocker, caplog):
+    try:
+        caplog.set_level(logging.DEBUG)
+
+        default_conf.update({"api_server": {"enabled": True,
+                                            "listen_ip_address": "127.0.0.1",
+                                            "listen_port": 8080,
+                                            "CORS_origins": ['http://example.com'],
+                                            "username": _TEST_USER,
+                                            "password": _TEST_PASS,
+                                            "ws_token": _TEST_WS_TOKEN
+                                            }})
+        mocker.patch('freqtrade.rpc.telegram.Updater')
+        mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api')
+        apiserver = ApiServer(default_conf)
+        apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf)))
+        apiserver.start_message_queue()
+        # Give the queue thread time to start
+        time.sleep(0.2)
+
+        # Test message_queue coro receives the message
+        test_message = {"type": "status", "data": "test"}
+        apiserver.send_msg(test_message)
+        time.sleep(0.1)  # Not sure how else to wait for the coro to receive the data
+        assert log_has("Found message of type: status", caplog)
+
+        # Test if exception logged when error occurs in sending
+        mocker.patch('freqtrade.rpc.api_server.ws.channel.ChannelManager.broadcast',
+                     side_effect=Exception)
+
+        apiserver.send_msg(test_message)
+        time.sleep(0.1)  # Not sure how else to wait for the coro to receive the data
+        assert log_has_re(r"Exception happened in background task.*", caplog)
+
+    finally:
+        apiserver.cleanup()
+        ApiServer.shutdown()

tests/rpc/test_rpc_emc.py

@@ -0,0 +1,467 @@
+"""
+Unit test file for rpc/external_message_consumer.py
+"""
+import asyncio
+import functools
+import logging
+from datetime import datetime, timezone
+from unittest.mock import MagicMock
+
+import pytest
+import websockets
+
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
+from tests.conftest import log_has, log_has_re, log_has_when
+
+
+_TEST_WS_TOKEN = "secret_Ws_t0ken"
+_TEST_WS_HOST = "127.0.0.1"
+_TEST_WS_PORT = 9989
+
+
+@pytest.fixture
+def patched_emc(default_conf, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "null",
+                    "port": 9891,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ]
+        }
+    })
+    dataprovider = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dataprovider)
+
+    try:
+        yield emc
+    finally:
+        emc.shutdown()
+
+
+def test_emc_start(patched_emc, caplog):
+    # Test if the message was printed
+    assert log_has_when("Starting ExternalMessageConsumer", caplog, "setup")
+    # Test if the thread and loop objects were created
+    assert patched_emc._thread and patched_emc._loop
+
+    # Test we call start again nothing happens
+    prev_thread = patched_emc._thread
+    patched_emc.start()
+    assert prev_thread == patched_emc._thread
+
+
+def test_emc_shutdown(patched_emc, caplog):
+    patched_emc.shutdown()
+
+    assert log_has("Stopping ExternalMessageConsumer", caplog)
+    # Test the loop has stopped
+    assert patched_emc._loop is None
+    # Test if the thread has stopped
+    assert patched_emc._thread is None
+
+    caplog.clear()
+    patched_emc.shutdown()
+
+    # Test func didn't run again as it was called once already
+    assert not log_has("Stopping ExternalMessageConsumer", caplog)
+
+
+def test_emc_init(patched_emc):
+    # Test the settings were set correctly
+    assert patched_emc.initial_candle_limit <= 1500
+    assert patched_emc.wait_timeout > 0
+    assert patched_emc.sleep_time > 0
+
+
+# Parametrize this?
+def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):
+    test_producer = {"name": "test", "url": "ws://test", "ws_token": "test"}
+    producer_name = test_producer['name']
+
+    caplog.set_level(logging.DEBUG)
+
+    # Test handle whitelist message
+    whitelist_message = {"type": "whitelist", "data": ["BTC/USDT"]}
+    patched_emc.handle_producer_message(test_producer, whitelist_message)
+
+    assert log_has(f"Received message of type `whitelist` from `{producer_name}`", caplog)
+    assert log_has(
+        f"Consumed message from `{producer_name}` of type `RPCMessageType.WHITELIST`", caplog)
+
+    # Test handle analyzed_df message
+    df_message = {
+        "type": "analyzed_df",
+        "data": {
+            "key": ("BTC/USDT", "5m", "spot"),
+            "df": ohlcv_history,
+            "la": datetime.now(timezone.utc)
+        }
+    }
+    patched_emc.handle_producer_message(test_producer, df_message)
+
+    assert log_has(f"Received message of type `analyzed_df` from `{producer_name}`", caplog)
+    assert log_has(
+        f"Consumed message from `{producer_name}` of type `RPCMessageType.ANALYZED_DF`", caplog)
+
+    # Test unhandled message
+    unhandled_message = {"type": "status", "data": "RUNNING"}
+    patched_emc.handle_producer_message(test_producer, unhandled_message)
+
+    assert log_has_re(r"Received unhandled message\: .*", caplog)
+
+    # Test malformed messages
+    caplog.clear()
+    malformed_message = {"type": "whitelist", "data": {"pair": "BTC/USDT"}}
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has_re(r"Invalid message .+", caplog)
+
+    malformed_message = {
+        "type": "analyzed_df",
+        "data": {
+            "key": "BTC/USDT",
+            "df": ohlcv_history,
+            "la": datetime.now(timezone.utc)
+        }
+    }
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has(f"Received message of type `analyzed_df` from `{producer_name}`", caplog)
+    assert log_has_re(r"Invalid message .+", caplog)
+
+    caplog.clear()
+    malformed_message = {"some": "stuff"}
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has_re(r"Invalid message .+", caplog)
+
+    caplog.clear()
+    malformed_message = {"type": "whitelist", "data": None}
+    patched_emc.handle_producer_message(test_producer, malformed_message)
+
+    assert log_has_re(r"Empty message .+", caplog)
+
+
+async def test_emc_create_connection_success(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+    lock = asyncio.Lock()
+
+    emc._running = True
+
+    async def eat(websocket):
+        emc._running = False
+
+    try:
+        async with websockets.serve(eat, _TEST_WS_HOST, _TEST_WS_PORT):
+            await emc._create_connection(test_producer, lock)
+
+        assert log_has_re(r"Producer connection success.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_create_connection_invalid_port(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": -1,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    try:
+        await asyncio.sleep(0.01)
+        assert log_has_re(r".+ is an invalid WebSocket URL .+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_create_connection_invalid_host(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "10000.1241..2121/",
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    try:
+        await asyncio.sleep(0.01)
+        assert log_has_re(r".+ is an invalid WebSocket URL .+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_create_connection_error(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 60,
+            "ping_timeout": 60,
+            "sleep_timeout": 60
+        }
+    })
+
+    # Test unexpected error
+    mocker.patch('websockets.connect', side_effect=RuntimeError)
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    try:
+        await asyncio.sleep(0.01)
+        assert log_has("Unexpected error has occurred:", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_valid(default_conf, caplog, mocker):
+    caplog.set_level(logging.DEBUG)
+
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 1,
+            "ping_timeout": 60,
+            "sleep_time": 60
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            return {"type": "whitelist", "data": ["BTC/USDT"]}
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Received message of type `whitelist`.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_invalid(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 1,
+            "ping_timeout": 60,
+            "sleep_time": 60
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            return {"type": ["BTC/USDT"]}
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Invalid message from.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_timeout(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 0.1,
+            "ping_timeout": 1,
+            "sleep_time": 1
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            await asyncio.sleep(0.2)
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Ping error.+", caplog)
+    finally:
+        emc.shutdown()
+
+
+async def test_emc_receive_messages_handle_error(default_conf, caplog, mocker):
+    default_conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": _TEST_WS_HOST,
+                    "port": _TEST_WS_PORT,
+                    "ws_token": _TEST_WS_TOKEN
+                }
+            ],
+            "wait_timeout": 1,
+            "ping_timeout": 1,
+            "sleep_time": 1
+        }
+    })
+
+    mocker.patch('freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start',
+                 MagicMock())
+
+    lock = asyncio.Lock()
+    test_producer = default_conf['external_message_consumer']['producers'][0]
+
+    dp = DataProvider(default_conf, None, None, None)
+    emc = ExternalMessageConsumer(default_conf, dp)
+
+    emc.handle_producer_message = MagicMock(side_effect=Exception)
+
+    loop = asyncio.get_event_loop()
+    def change_running(emc): emc._running = not emc._running
+
+    class TestChannel:
+        async def recv(self, *args, **kwargs):
+            return {"type": "whitelist", "data": ["BTC/USDT"]}
+
+        async def ping(self, *args, **kwargs):
+            return asyncio.Future()
+
+    try:
+        change_running(emc)
+        loop.call_soon(functools.partial(change_running, emc=emc))
+        await emc._receive_messages(TestChannel(), test_producer, lock)
+
+        assert log_has_re(r"Error handling producer message.+", caplog)
+    finally:
+        emc.shutdown()

tests/rpc/test_rpc_webhook.py

@@ -365,6 +365,14 @@ def test_exception_send_msg(default_conf, mocker, caplog):
     with pytest.raises(NotImplementedError):
         webhook.send_msg(msg)
 
+    # Test no failure for not implemented but known messagetypes
+    for e in RPCMessageType:
+        msg = {
+            'type': e,
+            'status': 'whatever'
+            }
+        webhook.send_msg(msg)
+
 
 def test__send_msg(default_conf, mocker, caplog):
     default_conf["webhook"] = get_webhook_dict()

tests/strategy/test_interface.py

@@ -288,7 +288,7 @@ def test_advise_all_indicators(default_conf, testdatadir) -> None:
     data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
                      fill_up_missing=True)
     processed = strategy.advise_all_indicators(data)
-    assert len(processed['UNITTEST/BTC']) == 102  # partial candle was removed
+    assert len(processed['UNITTEST/BTC']) == 103
 
 
 def test_populate_any_indicators(default_conf, testdatadir) -> None:
@@ -300,7 +300,7 @@ def test_populate_any_indicators(default_conf, testdatadir) -> None:
     processed = strategy.populate_any_indicators('UNITTEST/BTC', data, '5m')
     assert processed == data
     assert id(processed) == id(data)
-    assert len(processed['UNITTEST/BTC']) == 102  # partial candle was removed
+    assert len(processed['UNITTEST/BTC']) == 103
 
 
 def test_freqai_not_initialized(default_conf) -> None:

tests/test_configuration.py

@@ -1089,6 +1089,58 @@ def test__validate_pricing_rules(default_conf, caplog) -> None:
         validate_config_consistency(conf)
 
 
+def test__validate_consumers(default_conf, caplog) -> None:
+    conf = deepcopy(default_conf)
+    conf.update({
+            "external_message_consumer": {
+                "enabled": True,
+                "producers": []
+                }
+            })
+    with pytest.raises(OperationalException,
+                       match="You must specify at least 1 Producer to connect to."):
+        validate_config_consistency(conf)
+
+    conf = deepcopy(default_conf)
+    conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8081,
+                    "ws_token": "secret_ws_t0ken."
+                }, {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8080,
+                    "ws_token": "secret_ws_t0ken."
+                }
+            ]}
+        })
+    with pytest.raises(OperationalException,
+                       match="Producer names must be unique. Duplicate: default"):
+        validate_config_consistency(conf)
+
+    conf = deepcopy(default_conf)
+    conf.update({
+        "process_only_new_candles": True,
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8081,
+                    "ws_token": "secret_ws_t0ken."
+                }
+            ]}
+        })
+    validate_config_consistency(conf)
+    assert log_has_re("To receive best performance with external data.*", caplog)
+
+
 def test_load_config_test_comments() -> None:
     """
     Load config with comments

tests/test_freqtradebot.py

@@ -1319,9 +1319,9 @@ def test_create_stoploss_order_invalid_order(
     assert create_order_mock.call_args[1]['amount'] == trade.amount
 
     # Rpc is sending first buy, then sell
-    assert rpc_mock.call_count == 2
-    assert rpc_mock.call_args_list[1][0][0]['sell_reason'] == ExitType.EMERGENCY_EXIT.value
-    assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market'
+    assert rpc_mock.call_count == 3
+    assert rpc_mock.call_args_list[2][0][0]['sell_reason'] == ExitType.EMERGENCY_EXIT.value
+    assert rpc_mock.call_args_list[2][0][0]['order_type'] == 'market'
 
 
 @pytest.mark.parametrize("is_short", [False, True])
@@ -2439,7 +2439,7 @@ def test_manage_open_orders_entry_usercustom(
     # Trade should be closed since the function returns true
     freqtrade.manage_open_orders()
     assert cancel_order_wr_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 0
@@ -2478,7 +2478,7 @@ def test_manage_open_orders_entry(
     # check it does cancel buy orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 0
@@ -2608,7 +2608,7 @@ def test_check_handle_cancelled_buy(
     # check it does cancel buy orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 0
     assert log_has_re(
@@ -2639,7 +2639,7 @@ def test_manage_open_orders_buy_exception(
     # check it does cancel buy orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 0
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 1
@@ -2661,6 +2661,7 @@ def test_manage_open_orders_exit_usercustom(
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock()
     patch_exchange(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.get_min_pair_stake_amount', return_value=0.0)
     et_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit')
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -2673,7 +2674,6 @@ def test_manage_open_orders_exit_usercustom(
     open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime
     open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime
     open_trade_usdt.close_profit_abs = 0.001
-    open_trade_usdt.is_open = False
 
     Trade.query.session.add(open_trade_usdt)
     Trade.commit()
@@ -2686,8 +2686,7 @@ def test_manage_open_orders_exit_usercustom(
     # Return false - No impact
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 0
-    assert open_trade_usdt.is_open is False
+    assert rpc_mock.call_count == 1
     assert freqtrade.strategy.check_exit_timeout.call_count == 1
     assert freqtrade.strategy.check_entry_timeout.call_count == 0
 
@@ -2696,8 +2695,7 @@ def test_manage_open_orders_exit_usercustom(
     # Return Error - No impact
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 0
-    assert open_trade_usdt.is_open is False
+    assert rpc_mock.call_count == 1
     assert freqtrade.strategy.check_exit_timeout.call_count == 1
     assert freqtrade.strategy.check_entry_timeout.call_count == 0
 
@@ -2706,8 +2704,7 @@ def test_manage_open_orders_exit_usercustom(
     freqtrade.strategy.check_entry_timeout = MagicMock(return_value=True)
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
-    assert open_trade_usdt.is_open is True
+    assert rpc_mock.call_count == 2
     assert freqtrade.strategy.check_exit_timeout.call_count == 1
     assert freqtrade.strategy.check_entry_timeout.call_count == 0
 
@@ -2748,14 +2745,14 @@ def test_manage_open_orders_exit(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         fetch_order=MagicMock(return_value=limit_sell_order_old),
-        cancel_order=cancel_order_mock
+        cancel_order=cancel_order_mock,
+        get_min_pair_stake_amount=MagicMock(return_value=0),
     )
     freqtrade = FreqtradeBot(default_conf_usdt)
 
     open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime
     open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime
     open_trade_usdt.close_profit_abs = 0.001
-    open_trade_usdt.is_open = False
     open_trade_usdt.is_short = is_short
 
     Trade.query.session.add(open_trade_usdt)
@@ -2766,7 +2763,7 @@ def test_manage_open_orders_exit(
     # check it does cancel sell orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     assert open_trade_usdt.is_open is True
     # Custom user sell-timeout is never called
     assert freqtrade.strategy.check_exit_timeout.call_count == 0
@@ -2796,7 +2793,6 @@ def test_check_handle_cancelled_exit(
 
     open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime
     open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime
-    open_trade_usdt.is_open = False
     open_trade_usdt.is_short = is_short
 
     Trade.query.session.add(open_trade_usdt)
@@ -2805,7 +2801,7 @@ def test_check_handle_cancelled_exit(
     # check it does cancel sell orders over the time limit
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 0
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     assert open_trade_usdt.is_open is True
     exit_name = 'Buy' if is_short else 'Sell'
     assert log_has_re(f"{exit_name} order cancelled on exchange for Trade.*", caplog)
@@ -2843,7 +2839,7 @@ def test_manage_open_orders_partial(
     # note this is for a partially-complete buy order
     freqtrade.manage_open_orders()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     assert trades[0].amount == 23.0
@@ -2890,7 +2886,7 @@ def test_manage_open_orders_partial_fee(
     assert log_has_re(r"Applying fee on amount for Trade.*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2940,7 +2936,7 @@ def test_manage_open_orders_partial_except(
     assert log_has_re(r"Could not update trade amount: .*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -3004,6 +3000,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_order, is_
     trade.open_rate = 200
     trade.is_short = False
     trade.entry_side = "buy"
+    trade.amount = 100
     l_order['filled'] = 0.0
     l_order['status'] = 'open'
     trade.nr_of_successful_entries = 0
@@ -3092,6 +3089,7 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf_usdt, limit_order
     trade.entry_side = "buy"
     trade.open_order_id = "open_order_noop"
     trade.nr_of_successful_entries = 0
+    trade.amount = 100
     l_order['filled'] = 0.0
     l_order['status'] = 'open'
     reason = CANCEL_REASON['TIMEOUT']
@@ -3121,20 +3119,21 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None:
         amount=2,
         exchange='binance',
         open_rate=0.245441,
-        open_order_id="123456",
+        open_order_id="sell_123456",
         open_date=arrow.utcnow().shift(days=-2).datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         close_rate=0.555,
         close_date=arrow.utcnow().datetime,
         exit_reason="sell_reason_whatever",
+        stake_amount=0.245441 * 2,
     )
     trade.orders = [
-         Order(
+        Order(
             ft_order_side='buy',
             ft_pair=trade.pair,
-            ft_is_open=True,
-            order_id='123456',
+            ft_is_open=False,
+            order_id='buy_123456',
             status="closed",
             symbol=trade.pair,
             order_type="market",
@@ -3147,12 +3146,30 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None:
             order_date=trade.open_date,
             order_filled_date=trade.open_date,
              ),
+        Order(
+            ft_order_side='sell',
+            ft_pair=trade.pair,
+            ft_is_open=True,
+            order_id='sell_123456',
+            status="open",
+            symbol=trade.pair,
+            order_type="limit",
+            side="sell",
+            price=trade.open_rate,
+            average=trade.open_rate,
+            filled=0.0,
+            remaining=trade.amount,
+            cost=trade.open_rate * trade.amount,
+            order_date=trade.open_date,
+            order_filled_date=trade.open_date,
+             ),
     ]
-    order = {'id': "123456",
+    order = {'id': "sell_123456",
              'remaining': 1,
              'amount': 1,
              'status': "open"}
     reason = CANCEL_REASON['TIMEOUT']
+    send_msg_mock.reset_mock()
     assert freqtrade.handle_cancel_exit(trade, order, reason)
     assert cancel_order_mock.call_count == 1
     assert send_msg_mock.call_count == 1
@@ -3181,8 +3198,9 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None:
 def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.cancel_order_with_result', side_effect=InvalidOrderException())
+    mocker.patch('freqtrade.exchange.Exchange.get_min_pair_stake_amount', return_value=0.0)
+    mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result',
+                 side_effect=InvalidOrderException())
 
     freqtrade = FreqtradeBot(default_conf_usdt)
 
@@ -3592,7 +3610,7 @@ def test_execute_trade_exit_with_stoploss_on_exchange(
     trade.is_short = is_short
     assert trade
     assert cancel_order.call_count == 1
-    assert rpc_mock.call_count == 3
+    assert rpc_mock.call_count == 4
 
 
 @pytest.mark.parametrize("is_short", [False, True])
@@ -3662,11 +3680,11 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit(
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
     assert trade.exit_reason == ExitType.STOPLOSS_ON_EXCHANGE.value
-    assert rpc_mock.call_count == 3
-    assert rpc_mock.call_args_list[0][0][0]['type'] == RPCMessageType.ENTRY
-    assert rpc_mock.call_args_list[0][0][0]['amount'] > 20
-    assert rpc_mock.call_args_list[1][0][0]['type'] == RPCMessageType.ENTRY_FILL
-    assert rpc_mock.call_args_list[2][0][0]['type'] == RPCMessageType.EXIT_FILL
+    assert rpc_mock.call_count == 4
+    assert rpc_mock.call_args_list[1][0][0]['type'] == RPCMessageType.ENTRY
+    assert rpc_mock.call_args_list[1][0][0]['amount'] > 20
+    assert rpc_mock.call_args_list[2][0][0]['type'] == RPCMessageType.ENTRY_FILL
+    assert rpc_mock.call_args_list[3][0][0]['type'] == RPCMessageType.EXIT_FILL
 
 
 @pytest.mark.parametrize(

tests/test_integration.py

@@ -2,7 +2,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.enums import ExitCheckTuple, ExitType
+from freqtrade.enums import ExitCheckTuple, ExitType, TradingMode
 from freqtrade.persistence import Trade
 from freqtrade.persistence.models import Order
 from freqtrade.rpc.rpc import RPC
@@ -455,10 +455,12 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None:
     assert pytest.approx(trade.orders[-1].amount) == 61.538461232
 
 
-def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog) -> None:
+@pytest.mark.parametrize('leverage', [1, 2])
+def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog, leverage) -> None:
     default_conf_usdt['position_adjustment_enable'] = True
 
     freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
+    freqtrade.trading_mode = TradingMode.FUTURES
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
@@ -467,15 +469,17 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog) -> Non
         price_to_precision=lambda s, x, y: y,
         get_min_pair_stake_amount=MagicMock(return_value=10),
     )
+    mocker.patch("freqtrade.exchange.Exchange.get_max_leverage", return_value=10)
 
     patch_get_signal(freqtrade)
+    freqtrade.strategy.leverage = MagicMock(return_value=leverage)
     freqtrade.enter_positions()
 
     assert len(Trade.get_trades().all()) == 1
     trade = Trade.get_trades().first()
     assert len(trade.orders) == 1
     assert pytest.approx(trade.stake_amount) == 60
-    assert pytest.approx(trade.amount) == 30.0
+    assert pytest.approx(trade.amount) == 30.0 * leverage
     assert trade.open_rate == 2.0
 
     # Too small size
@@ -484,8 +488,9 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog) -> Non
     trade = Trade.get_trades().first()
     assert len(trade.orders) == 1
     assert pytest.approx(trade.stake_amount) == 60
-    assert pytest.approx(trade.amount) == 30.0
-    assert log_has_re("Remaining amount of 1.6.* would be smaller than the minimum of 10.", caplog)
+    assert pytest.approx(trade.amount) == 30.0 * leverage
+    assert log_has_re(
+        r"Remaining amount of \d\.\d+.* would be smaller than the minimum of 10.", caplog)
 
     freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-20)
 
@@ -494,7 +499,7 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog) -> Non
     assert len(trade.orders) == 2
     assert trade.orders[-1].ft_order_side == 'sell'
     assert pytest.approx(trade.stake_amount) == 40.198
-    assert pytest.approx(trade.amount) == 20.099
+    assert pytest.approx(trade.amount) == 20.099 * leverage
     assert trade.open_rate == 2.0
     assert trade.is_open
     caplog.clear()

tests/test_misc.py

@@ -7,10 +7,11 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.misc import (decimals_per_coin, deep_merge_dicts, file_dump_json, file_load_json,
-                            format_ms_time, pair_to_filename, parse_db_uri_for_logging, plural,
-                            render_template, render_template_with_fallback, round_coin_value,
-                            safe_value_fallback, safe_value_fallback2, shorten_date)
+from freqtrade.misc import (dataframe_to_json, decimals_per_coin, deep_merge_dicts, file_dump_json,
+                            file_load_json, format_ms_time, json_to_dataframe, pair_to_filename,
+                            parse_db_uri_for_logging, plural, render_template,
+                            render_template_with_fallback, round_coin_value, safe_value_fallback,
+                            safe_value_fallback2, shorten_date)
 
 
 def test_decimals_per_coin():
@@ -219,3 +220,14 @@ def test_deep_merge_dicts():
 
     res2['first']['rows']['test'] = 'asdf'
     assert deep_merge_dicts(a, deepcopy(b), allow_null_overrides=False) == res2
+
+
+def test_dataframe_json(ohlcv_history):
+    from pandas.testing import assert_frame_equal
+    json = dataframe_to_json(ohlcv_history)
+    dataframe = json_to_dataframe(json)
+
+    assert list(ohlcv_history.columns) == list(dataframe.columns)
+    assert len(ohlcv_history) == len(dataframe)
+
+    assert_frame_equal(ohlcv_history, dataframe)