Split datahandler and history tests

tests/data/test_datahandler.py

@@ -0,0 +1,353 @@
+# pragma pylint: disable=missing-docstring, protected-access, C0103
+
+import re
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+from pandas import DataFrame
+
+from freqtrade.configuration import TimeRange
+from freqtrade.constants import AVAILABLE_DATAHANDLERS
+from freqtrade.data.history.hdf5datahandler import HDF5DataHandler
+from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass
+from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
+from freqtrade.enums import CandleType, TradingMode
+from tests.conftest import log_has
+
+
+def test_datahandler_ohlcv_get_pairs(testdatadir):
+    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC',
+                          'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC',
+                          'DASH/BTC', 'XRP/ETH'}
+
+    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m', candle_type=CandleType.SPOT)
+    assert set(pairs) == {'UNITTEST/BTC'}
+
+    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
+    assert set(pairs) == {'UNITTEST/BTC'}
+
+    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
+    assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'}
+
+    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES)
+    assert set(pairs) == {'XRP/USDT'}
+
+    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
+    assert set(pairs) == {'UNITTEST/USDT:USDT'}
+
+
+@pytest.mark.parametrize('filename,pair,timeframe,candletype', [
+    ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''),
+    ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''),
+    ('BTC-PERP-1h.h5', 'BTC-PERP', '1h', ''),
+    ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''),
+    ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'),
+    ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'),
+    ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'),
+    ('BTC-PERP-1h-index.h5', 'BTC-PERP', '1h', 'index'),
+    ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'),
+])
+def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype):
+    regex = JsonDataHandler._OHLCV_REGEX
+
+    match = re.search(regex, filename)
+    assert len(match.groups()) > 1
+    assert match[1] == pair
+    assert match[2] == timeframe
+    assert match[3] == candletype
+
+
+@pytest.mark.parametrize('input,expected', [
+    ('XMR_USDT', 'XMR/USDT'),
+    ('BTC_USDT', 'BTC/USDT'),
+    ('USDT_BUSD', 'USDT/BUSD'),
+    ('BTC_USDT_USDT', 'BTC/USDT:USDT'),  # Futures
+    ('XRP_USDT_USDT', 'XRP/USDT:USDT'),  # futures
+    ('BTC-PERP', 'BTC-PERP'),
+    ('BTC-PERP_USDT', 'BTC-PERP:USDT'),  # potential FTX case
+    ('UNITTEST_USDT', 'UNITTEST/USDT'),
+])
+def test_rebuild_pair_from_filename(input, expected):
+
+    assert IDataHandler.rebuild_pair_from_filename(input) == expected
+
+
+def test_datahandler_ohlcv_get_available_data(testdatadir):
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {
+        ('UNITTEST/BTC', '5m', CandleType.SPOT),
+        ('ETH/BTC', '5m', CandleType.SPOT),
+        ('XLM/BTC', '5m', CandleType.SPOT),
+        ('TRX/BTC', '5m', CandleType.SPOT),
+        ('LTC/BTC', '5m', CandleType.SPOT),
+        ('XMR/BTC', '5m', CandleType.SPOT),
+        ('ZEC/BTC', '5m', CandleType.SPOT),
+        ('UNITTEST/BTC', '1m', CandleType.SPOT),
+        ('ADA/BTC', '5m', CandleType.SPOT),
+        ('ETC/BTC', '5m', CandleType.SPOT),
+        ('NXT/BTC', '5m', CandleType.SPOT),
+        ('DASH/BTC', '5m', CandleType.SPOT),
+        ('XRP/ETH', '1m', CandleType.SPOT),
+        ('XRP/ETH', '5m', CandleType.SPOT),
+        ('UNITTEST/BTC', '30m', CandleType.SPOT),
+        ('UNITTEST/BTC', '8m', CandleType.SPOT),
+        ('NOPAIR/XXX', '4m', CandleType.SPOT),
+    }
+
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.FUTURES)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {
+        ('UNITTEST/USDT', '1h', 'mark'),
+        ('XRP/USDT', '1h', 'futures'),
+        ('XRP/USDT', '1h', 'mark'),
+        ('XRP/USDT', '8h', 'mark'),
+        ('XRP/USDT', '8h', 'funding_rate'),
+    }
+
+    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {('UNITTEST/BTC', '8m', CandleType.SPOT)}
+    paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)}
+
+
+def test_jsondatahandler_trades_get_pairs(testdatadir):
+    pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
+
+
+def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = JsonGzDataHandler(testdatadir)
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 2
+
+
+def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
+    dh = JsonDataHandler(testdatadir)
+    df = dh.ohlcv_load('XRP/ETH', '5m', 'spot')
+    assert len(df) == 711
+
+    df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark")
+    assert len(df_mark) == 99
+
+    df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot')
+    assert len(df_no_mark) == 0
+
+    # Failure case (empty array)
+    df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', 'spot')
+    assert len(df1) == 0
+    assert log_has("Could not load data for NOPAIR/XXX.", caplog)
+    assert df.columns.equals(df1.columns)
+
+
+def test_jsondatahandler_trades_load(testdatadir, caplog):
+    dh = JsonGzDataHandler(testdatadir)
+    logmsg = "Old trades format detected - converting"
+    dh.trades_load('XRP/ETH')
+    assert not log_has(logmsg, caplog)
+
+    # Test conversation is happening
+    dh.trades_load('XRP/OLD')
+    assert log_has(logmsg, caplog)
+
+
+def test_jsondatahandler_trades_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = JsonGzDataHandler(testdatadir)
+    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
+
+
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
+def test_datahandler_ohlcv_append(datahandler, testdatadir, ):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.SPOT)
+    with pytest.raises(NotImplementedError):
+        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.MARK)
+
+
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
+def test_datahandler_trades_append(datahandler, testdatadir):
+    dh = get_datahandler(testdatadir, datahandler)
+    with pytest.raises(NotImplementedError):
+        dh.trades_append('UNITTEST/ETH', [])
+
+
+def test_hdf5datahandler_trades_get_pairs(testdatadir):
+    pairs = HDF5DataHandler.trades_get_pairs(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'XRP/ETH'}
+
+
+def test_hdf5datahandler_trades_load(testdatadir):
+    dh = HDF5DataHandler(testdatadir)
+    trades = dh.trades_load('XRP/ETH')
+    assert isinstance(trades, list)
+
+    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    assert trades1 == []
+    # data goes from 2019-10-11 - 2019-10-13
+    timerange = TimeRange.parse_timerange('20191011-20191012')
+
+    trades2 = dh._trades_load('XRP/ETH', timerange)
+    assert len(trades) > len(trades2)
+    # Check that ID is None (If it's nan, it's wrong)
+    assert trades2[0][2] is None
+
+    # unfiltered load has trades before starttime
+    assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0
+    # filtered list does not have trades before starttime
+    assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0
+    # unfiltered load has trades after endtime
+    assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0
+    # filtered list does not have trades after endtime
+    assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0
+
+
+def test_hdf5datahandler_trades_store(testdatadir, tmpdir):
+    tmpdir1 = Path(tmpdir)
+    dh = HDF5DataHandler(testdatadir)
+    trades = dh.trades_load('XRP/ETH')
+
+    dh1 = HDF5DataHandler(tmpdir1)
+    dh1.trades_store('XRP/NEW', trades)
+    file = tmpdir1 / 'XRP_NEW-trades.h5'
+    assert file.is_file()
+    # Load trades back
+    trades_new = dh1.trades_load('XRP/NEW')
+
+    assert len(trades_new) == len(trades)
+    assert trades[0][0] == trades_new[0][0]
+    assert trades[0][1] == trades_new[0][1]
+    # assert trades[0][2] == trades_new[0][2]  # This is nan - so comparison does not make sense
+    assert trades[0][3] == trades_new[0][3]
+    assert trades[0][4] == trades_new[0][4]
+    assert trades[0][5] == trades_new[0][5]
+    assert trades[0][6] == trades_new[0][6]
+    assert trades[-1][0] == trades_new[-1][0]
+    assert trades[-1][1] == trades_new[-1][1]
+    # assert trades[-1][2] == trades_new[-1][2]  # This is nan - so comparison does not make sense
+    assert trades[-1][3] == trades_new[-1][3]
+    assert trades[-1][4] == trades_new[-1][4]
+    assert trades[-1][5] == trades_new[-1][5]
+    assert trades[-1][6] == trades_new[-1][6]
+
+
+def test_hdf5datahandler_trades_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = HDF5DataHandler(testdatadir)
+    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
+
+
+@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
+    # Data goes from 2018-01-10 - 2018-01-30
+    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
+    # Mark data goes from to 2021-11-15 2021-11-19
+    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
+])
+def test_hdf5datahandler_ohlcv_load_and_resave(
+    testdatadir,
+    tmpdir,
+    pair,
+    timeframe,
+    candle_type,
+    candle_append,
+    startdt, enddt
+):
+    tmpdir1 = Path(tmpdir)
+    tmpdir2 = tmpdir1
+    if candle_type not in ('', 'spot'):
+        tmpdir2 = tmpdir1 / 'futures'
+        tmpdir2.mkdir()
+    dh = HDF5DataHandler(testdatadir)
+    ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
+    assert isinstance(ohlcv, DataFrame)
+    assert len(ohlcv) > 0
+
+    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5"
+    assert not file.is_file()
+
+    dh1 = HDF5DataHandler(tmpdir1)
+    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
+    assert file.is_file()
+
+    assert not ohlcv[ohlcv['date'] < startdt].empty
+
+    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
+
+    # Call private function to ensure timerange is filtered in hdf5
+    ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type)
+    ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
+    assert len(ohlcv) == len(ohlcv1)
+    assert ohlcv.equals(ohlcv1)
+    assert ohlcv[ohlcv['date'] < startdt].empty
+    assert ohlcv[ohlcv['date'] > enddt].empty
+
+    # Try loading inexisting file
+    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
+    assert ohlcv.empty
+
+
+def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = HDF5DataHandler(testdatadir)
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
+    assert unlinkmock.call_count == 2
+
+
+def test_gethandlerclass():
+    cl = get_datahandlerclass('json')
+    assert cl == JsonDataHandler
+    assert issubclass(cl, IDataHandler)
+    cl = get_datahandlerclass('jsongz')
+    assert cl == JsonGzDataHandler
+    assert issubclass(cl, IDataHandler)
+    assert issubclass(cl, JsonDataHandler)
+    cl = get_datahandlerclass('hdf5')
+    assert cl == HDF5DataHandler
+    assert issubclass(cl, IDataHandler)
+    with pytest.raises(ValueError, match=r"No datahandler for .*"):
+        get_datahandlerclass('DeadBeef')
+
+
+def test_get_datahandler(testdatadir):
+    dh = get_datahandler(testdatadir, 'json')
+    assert type(dh) == JsonDataHandler
+    dh = get_datahandler(testdatadir, 'jsongz')
+    assert type(dh) == JsonGzDataHandler
+    dh1 = get_datahandler(testdatadir, 'jsongz', dh)
+    assert id(dh1) == id(dh)
+
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert type(dh) == HDF5DataHandler

tests/data/test_history.py

@@ -1,7 +1,6 @@
 # pragma pylint: disable=missing-docstring, protected-access, C0103
 
 import json
-import re
 import uuid
 from pathlib import Path
 from shutil import copyfile
@@ -13,18 +12,17 @@ from pandas import DataFrame
 from pandas.testing import assert_frame_equal
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import AVAILABLE_DATAHANDLERS, DATETIME_PRINT_FORMAT
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data.converter import ohlcv_to_dataframe
-from freqtrade.data.history.hdf5datahandler import HDF5DataHandler
 from freqtrade.data.history.history_utils import (_download_pair_history, _download_trades_history,
                                                   _load_cached_data_for_updating,
                                                   convert_trades_to_ohlcv, get_timerange, load_data,
                                                   load_pair_history, refresh_backtest_ohlcv_data,
                                                   refresh_backtest_trades_data, refresh_data,
                                                   validate_backtest_data)
-from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass
+from freqtrade.data.history.idatahandler import get_datahandler
 from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
-from freqtrade.enums import CandleType, TradingMode
+from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.resolvers import StrategyResolver
@@ -32,25 +30,6 @@ from tests.conftest import (CURRENT_TEST_STRATEGY, get_patched_exchange, log_has
                             patch_exchange)
 
 
-# Change this if modifying UNITTEST/BTC testdatafile
-_BTC_UNITTEST_LENGTH = 13681
-
-
-def _backup_file(file: Path, copy_file: bool = False) -> None:
-    """
-    Backup existing file to avoid deleting the user file
-    :param file: complete path to the file
-    :param copy_file: keep file in place too.
-    :return: None
-    """
-    file_swp = str(file) + '.swp'
-    if file.is_file():
-        file.rename(file_swp)
-
-        if copy_file:
-            copyfile(file_swp, file)
-
-
 def _clean_test_file(file: Path) -> None:
     """
     Backup existing file to avoid deleting the user file
@@ -67,7 +46,7 @@ def _clean_test_file(file: Path) -> None:
         file_swp.rename(file)
 
 
-def test_load_data_30min_timeframe(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_30min_timeframe(caplog, testdatadir) -> None:
     ld = load_pair_history(pair='UNITTEST/BTC', timeframe='30m', datadir=testdatadir)
     assert isinstance(ld, DataFrame)
     assert not log_has(
@@ -76,7 +55,7 @@ def test_load_data_30min_timeframe(mocker, caplog, default_conf, testdatadir) ->
     )
 
 
-def test_load_data_7min_timeframe(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_7min_timeframe(caplog, testdatadir) -> None:
     ld = load_pair_history(pair='UNITTEST/BTC', timeframe='7m', datadir=testdatadir)
     assert isinstance(ld, DataFrame)
     assert ld.empty
@@ -108,7 +87,7 @@ def test_load_data_mark(ohlcv_history, mocker, caplog, testdatadir) -> None:
     )
 
 
-def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None:
+def test_load_data_startup_candles(mocker, testdatadir) -> None:
     ltfmock = mocker.patch(
         'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load',
         MagicMock(return_value=DataFrame()))
@@ -405,7 +384,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None:
                    caplog)
 
 
-def test_init(default_conf, mocker) -> None:
+def test_init(default_conf) -> None:
     assert {} == load_data(
         datadir=Path(''),
         pairs=[],
@@ -685,340 +664,3 @@ def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog):
     convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
                             datadir=tmpdir1, timerange=tr, erase=True)
     assert log_has('Could not convert NoDatapair to OHLCV.', caplog)
-
-
-def test_datahandler_ohlcv_get_pairs(testdatadir):
-    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC',
-                          'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC',
-                          'DASH/BTC', 'XRP/ETH'}
-
-    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m', candle_type=CandleType.SPOT)
-    assert set(pairs) == {'UNITTEST/BTC'}
-
-    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT)
-    assert set(pairs) == {'UNITTEST/BTC'}
-
-    pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'}
-
-    pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES)
-    assert set(pairs) == {'XRP/USDT'}
-
-    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT:USDT'}
-
-
-@pytest.mark.parametrize('filename,pair,timeframe,candletype', [
-    ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''),
-    ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''),
-    ('BTC-PERP-1h.h5', 'BTC-PERP', '1h', ''),
-    ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''),
-    ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'),
-    ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'),
-    ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'),
-    ('BTC-PERP-1h-index.h5', 'BTC-PERP', '1h', 'index'),
-    ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'),
-])
-def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype):
-    regex = JsonDataHandler._OHLCV_REGEX
-
-    match = re.search(regex, filename)
-    assert len(match.groups()) > 1
-    assert match[1] == pair
-    assert match[2] == timeframe
-    assert match[3] == candletype
-
-
-@pytest.mark.parametrize('input,expected', [
-    ('XMR_USDT', 'XMR/USDT'),
-    ('BTC_USDT', 'BTC/USDT'),
-    ('USDT_BUSD', 'USDT/BUSD'),
-    ('BTC_USDT_USDT', 'BTC/USDT:USDT'),  # Futures
-    ('XRP_USDT_USDT', 'XRP/USDT:USDT'),  # futures
-    ('BTC-PERP', 'BTC-PERP'),
-    ('BTC-PERP_USDT', 'BTC-PERP:USDT'),  # potential FTX case
-    ('UNITTEST_USDT', 'UNITTEST/USDT'),
-])
-def test_rebuild_pair_from_filename(input, expected):
-
-    assert IDataHandler.rebuild_pair_from_filename(input) == expected
-
-
-def test_datahandler_ohlcv_get_available_data(testdatadir):
-    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    # Convert to set to avoid failures due to sorting
-    assert set(paircombs) == {
-        ('UNITTEST/BTC', '5m', CandleType.SPOT),
-        ('ETH/BTC', '5m', CandleType.SPOT),
-        ('XLM/BTC', '5m', CandleType.SPOT),
-        ('TRX/BTC', '5m', CandleType.SPOT),
-        ('LTC/BTC', '5m', CandleType.SPOT),
-        ('XMR/BTC', '5m', CandleType.SPOT),
-        ('ZEC/BTC', '5m', CandleType.SPOT),
-        ('UNITTEST/BTC', '1m', CandleType.SPOT),
-        ('ADA/BTC', '5m', CandleType.SPOT),
-        ('ETC/BTC', '5m', CandleType.SPOT),
-        ('NXT/BTC', '5m', CandleType.SPOT),
-        ('DASH/BTC', '5m', CandleType.SPOT),
-        ('XRP/ETH', '1m', CandleType.SPOT),
-        ('XRP/ETH', '5m', CandleType.SPOT),
-        ('UNITTEST/BTC', '30m', CandleType.SPOT),
-        ('UNITTEST/BTC', '8m', CandleType.SPOT),
-        ('NOPAIR/XXX', '4m', CandleType.SPOT),
-    }
-
-    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.FUTURES)
-    # Convert to set to avoid failures due to sorting
-    assert set(paircombs) == {
-        ('UNITTEST/USDT', '1h', 'mark'),
-        ('XRP/USDT', '1h', 'futures'),
-        ('XRP/USDT', '1h', 'mark'),
-        ('XRP/USDT', '8h', 'mark'),
-        ('XRP/USDT', '8h', 'funding_rate'),
-    }
-
-    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    assert set(paircombs) == {('UNITTEST/BTC', '8m', CandleType.SPOT)}
-    paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
-    assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)}
-
-
-def test_jsondatahandler_trades_get_pairs(testdatadir):
-    pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
-
-
-def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = JsonGzDataHandler(testdatadir)
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 2
-
-
-def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
-    dh = JsonDataHandler(testdatadir)
-    df = dh.ohlcv_load('XRP/ETH', '5m', 'spot')
-    assert len(df) == 711
-
-    df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark")
-    assert len(df_mark) == 99
-
-    df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot')
-    assert len(df_no_mark) == 0
-
-    # Failure case (empty array)
-    df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', 'spot')
-    assert len(df1) == 0
-    assert log_has("Could not load data for NOPAIR/XXX.", caplog)
-    assert df.columns.equals(df1.columns)
-
-
-def test_jsondatahandler_trades_load(testdatadir, caplog):
-    dh = JsonGzDataHandler(testdatadir)
-    logmsg = "Old trades format detected - converting"
-    dh.trades_load('XRP/ETH')
-    assert not log_has(logmsg, caplog)
-
-    # Test conversation is happening
-    dh.trades_load('XRP/OLD')
-    assert log_has(logmsg, caplog)
-
-
-def test_jsondatahandler_trades_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = JsonGzDataHandler(testdatadir)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 1
-
-
-@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-def test_datahandler_ohlcv_append(datahandler, testdatadir, ):
-    dh = get_datahandler(testdatadir, datahandler)
-    with pytest.raises(NotImplementedError):
-        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.SPOT)
-    with pytest.raises(NotImplementedError):
-        dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.MARK)
-
-
-@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-def test_datahandler_trades_append(datahandler, testdatadir):
-    dh = get_datahandler(testdatadir, datahandler)
-    with pytest.raises(NotImplementedError):
-        dh.trades_append('UNITTEST/ETH', [])
-
-
-def test_hdf5datahandler_trades_get_pairs(testdatadir):
-    pairs = HDF5DataHandler.trades_get_pairs(testdatadir)
-    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH'}
-
-
-def test_hdf5datahandler_trades_load(testdatadir):
-    dh = HDF5DataHandler(testdatadir)
-    trades = dh.trades_load('XRP/ETH')
-    assert isinstance(trades, list)
-
-    trades1 = dh.trades_load('UNITTEST/NONEXIST')
-    assert trades1 == []
-    # data goes from 2019-10-11 - 2019-10-13
-    timerange = TimeRange.parse_timerange('20191011-20191012')
-
-    trades2 = dh._trades_load('XRP/ETH', timerange)
-    assert len(trades) > len(trades2)
-    # Check that ID is None (If it's nan, it's wrong)
-    assert trades2[0][2] is None
-
-    # unfiltered load has trades before starttime
-    assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0
-    # filtered list does not have trades before starttime
-    assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0
-    # unfiltered load has trades after endtime
-    assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0
-    # filtered list does not have trades after endtime
-    assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0
-
-
-def test_hdf5datahandler_trades_store(testdatadir, tmpdir):
-    tmpdir1 = Path(tmpdir)
-    dh = HDF5DataHandler(testdatadir)
-    trades = dh.trades_load('XRP/ETH')
-
-    dh1 = HDF5DataHandler(tmpdir1)
-    dh1.trades_store('XRP/NEW', trades)
-    file = tmpdir1 / 'XRP_NEW-trades.h5'
-    assert file.is_file()
-    # Load trades back
-    trades_new = dh1.trades_load('XRP/NEW')
-
-    assert len(trades_new) == len(trades)
-    assert trades[0][0] == trades_new[0][0]
-    assert trades[0][1] == trades_new[0][1]
-    # assert trades[0][2] == trades_new[0][2]  # This is nan - so comparison does not make sense
-    assert trades[0][3] == trades_new[0][3]
-    assert trades[0][4] == trades_new[0][4]
-    assert trades[0][5] == trades_new[0][5]
-    assert trades[0][6] == trades_new[0][6]
-    assert trades[-1][0] == trades_new[-1][0]
-    assert trades[-1][1] == trades_new[-1][1]
-    # assert trades[-1][2] == trades_new[-1][2]  # This is nan - so comparison does not make sense
-    assert trades[-1][3] == trades_new[-1][3]
-    assert trades[-1][4] == trades_new[-1][4]
-    assert trades[-1][5] == trades_new[-1][5]
-    assert trades[-1][6] == trades_new[-1][6]
-
-
-def test_hdf5datahandler_trades_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = HDF5DataHandler(testdatadir)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
-    assert unlinkmock.call_count == 1
-
-
-@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [
-    # Data goes from 2018-01-10 - 2018-01-30
-    ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
-    # Mark data goes from to 2021-11-15 2021-11-19
-    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
-])
-def test_hdf5datahandler_ohlcv_load_and_resave(
-    testdatadir,
-    tmpdir,
-    pair,
-    timeframe,
-    candle_type,
-    candle_append,
-    startdt, enddt
-):
-    tmpdir1 = Path(tmpdir)
-    tmpdir2 = tmpdir1
-    if candle_type not in ('', 'spot'):
-        tmpdir2 = tmpdir1 / 'futures'
-        tmpdir2.mkdir()
-    dh = HDF5DataHandler(testdatadir)
-    ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
-    assert isinstance(ohlcv, DataFrame)
-    assert len(ohlcv) > 0
-
-    file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5"
-    assert not file.is_file()
-
-    dh1 = HDF5DataHandler(tmpdir1)
-    dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
-    assert file.is_file()
-
-    assert not ohlcv[ohlcv['date'] < startdt].empty
-
-    timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}")
-
-    # Call private function to ensure timerange is filtered in hdf5
-    ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type)
-    ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type)
-    assert len(ohlcv) == len(ohlcv1)
-    assert ohlcv.equals(ohlcv1)
-    assert ohlcv[ohlcv['date'] < startdt].empty
-    assert ohlcv[ohlcv['date'] > enddt].empty
-
-    # Try loading inexisting file
-    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type)
-    assert ohlcv.empty
-
-
-def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
-    dh = HDF5DataHandler(testdatadir)
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 0
-
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '')
-    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark')
-    assert unlinkmock.call_count == 2
-
-
-def test_gethandlerclass():
-    cl = get_datahandlerclass('json')
-    assert cl == JsonDataHandler
-    assert issubclass(cl, IDataHandler)
-    cl = get_datahandlerclass('jsongz')
-    assert cl == JsonGzDataHandler
-    assert issubclass(cl, IDataHandler)
-    assert issubclass(cl, JsonDataHandler)
-    cl = get_datahandlerclass('hdf5')
-    assert cl == HDF5DataHandler
-    assert issubclass(cl, IDataHandler)
-    with pytest.raises(ValueError, match=r"No datahandler for .*"):
-        get_datahandlerclass('DeadBeef')
-
-
-def test_get_datahandler(testdatadir):
-    dh = get_datahandler(testdatadir, 'json')
-    assert type(dh) == JsonDataHandler
-    dh = get_datahandler(testdatadir, 'jsongz')
-    assert type(dh) == JsonGzDataHandler
-    dh1 = get_datahandler(testdatadir, 'jsongz', dh)
-    assert id(dh1) == id(dh)
-
-    dh = get_datahandler(testdatadir, 'hdf5')
-    assert type(dh) == HDF5DataHandler

tests/optimize/test_optimize_reports.py

@@ -1,6 +1,7 @@
 import re
 from datetime import timedelta
 from pathlib import Path
+from shutil import copyfile
 
 import joblib
 import pandas as pd
@@ -25,7 +26,22 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene
                                                  text_table_exit_reason, text_table_strategy)
 from freqtrade.resolvers.strategy_resolver import StrategyResolver
 from tests.conftest import CURRENT_TEST_STRATEGY
-from tests.data.test_history import _backup_file, _clean_test_file
+from tests.data.test_history import _clean_test_file
+
+
+def _backup_file(file: Path, copy_file: bool = False) -> None:
+    """
+    Backup existing file to avoid deleting the user file
+    :param file: complete path to the file
+    :param copy_file: keep file in place too.
+    :return: None
+    """
+    file_swp = str(file) + '.swp'
+    if file.is_file():
+        file.rename(file_swp)
+
+        if copy_file:
+            copyfile(file_swp, file)
 
 
 def test_text_table_bt_results():