Move tests out of freqtrade module

tests/data/test_btanalysis.py

@@ -0,0 +1,136 @@
+from unittest.mock import MagicMock
+
+import pytest
+from arrow import Arrow
+from pandas import DataFrame, to_datetime
+
+from freqtrade.configuration import TimeRange
+from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
+                                       combine_tickers_with_mean,
+                                       create_cum_profit,
+                                       extract_trades_of_period,
+                                       load_backtest_data, load_trades,
+                                       load_trades_from_db)
+from freqtrade.data.history import load_data, load_pair_history
+from freqtrade.tests.test_persistence import create_mock_trades
+
+
+def test_load_backtest_data(testdatadir):
+
+    filename = testdatadir / "backtest-result_test.json"
+    bt_data = load_backtest_data(filename)
+    assert isinstance(bt_data, DataFrame)
+    assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"]
+    assert len(bt_data) == 179
+
+    # Test loading from string (must yield same result)
+    bt_data2 = load_backtest_data(str(filename))
+    assert bt_data.equals(bt_data2)
+
+    with pytest.raises(ValueError, match=r"File .* does not exist\."):
+        load_backtest_data(str("filename") + "nofile")
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_load_trades_db(default_conf, fee, mocker):
+
+    create_mock_trades(fee)
+    # remove init so it does not init again
+    init_mock = mocker.patch('freqtrade.persistence.init', MagicMock())
+
+    trades = load_trades_from_db(db_url=default_conf['db_url'])
+    assert init_mock.call_count == 1
+    assert len(trades) == 3
+    assert isinstance(trades, DataFrame)
+    assert "pair" in trades.columns
+    assert "open_time" in trades.columns
+    assert "profitperc" in trades.columns
+
+    for col in BT_DATA_COLUMNS:
+        if col not in ['index', 'open_at_end']:
+            assert col in trades.columns
+
+
+def test_extract_trades_of_period(testdatadir):
+    pair = "UNITTEST/BTC"
+    timerange = TimeRange(None, 'line', 0, -1000)
+
+    data = load_pair_history(pair=pair, ticker_interval='1m',
+                             datadir=testdatadir, timerange=timerange)
+
+    # timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00
+    trades = DataFrame(
+        {'pair': [pair, pair, pair, pair],
+         'profit_percent': [0.0, 0.1, -0.2, -0.5],
+         'profit_abs': [0.0, 1, -2, -5],
+         'open_time': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime,
+                                   Arrow(2017, 11, 14, 9, 41, 0).datetime,
+                                   Arrow(2017, 11, 14, 14, 20, 0).datetime,
+                                   Arrow(2017, 11, 15, 3, 40, 0).datetime,
+                                   ], utc=True
+                                  ),
+         'close_time': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime,
+                                    Arrow(2017, 11, 14, 10, 41, 0).datetime,
+                                    Arrow(2017, 11, 14, 15, 25, 0).datetime,
+                                    Arrow(2017, 11, 15, 3, 55, 0).datetime,
+                                    ], utc=True)
+         })
+    trades1 = extract_trades_of_period(data, trades)
+    # First and last trade are dropped as they are out of range
+    assert len(trades1) == 2
+    assert trades1.iloc[0].open_time == Arrow(2017, 11, 14, 9, 41, 0).datetime
+    assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime
+    assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime
+    assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime
+
+
+def test_load_trades(default_conf, mocker):
+    db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock())
+    bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock())
+
+    load_trades("DB",
+                db_url=default_conf.get('db_url'),
+                exportfilename=default_conf.get('exportfilename'),
+                )
+
+    assert db_mock.call_count == 1
+    assert bt_mock.call_count == 0
+
+    db_mock.reset_mock()
+    bt_mock.reset_mock()
+    default_conf['exportfilename'] = "testfile.json"
+    load_trades("file",
+                db_url=default_conf.get('db_url'),
+                exportfilename=default_conf.get('exportfilename'),)
+
+    assert db_mock.call_count == 0
+    assert bt_mock.call_count == 1
+
+
+def test_combine_tickers_with_mean(testdatadir):
+    pairs = ["ETH/BTC", "XLM/BTC"]
+    tickers = load_data(datadir=testdatadir,
+                        pairs=pairs,
+                        ticker_interval='5m'
+                        )
+    df = combine_tickers_with_mean(tickers)
+    assert isinstance(df, DataFrame)
+    assert "ETH/BTC" in df.columns
+    assert "XLM/BTC" in df.columns
+    assert "mean" in df.columns
+
+
+def test_create_cum_profit(testdatadir):
+    filename = testdatadir / "backtest-result_test.json"
+    bt_data = load_backtest_data(filename)
+    timerange = TimeRange.parse_timerange("20180110-20180112")
+
+    df = load_pair_history(pair="POWR/BTC", ticker_interval='5m',
+                           datadir=testdatadir, timerange=timerange)
+
+    cum_profits = create_cum_profit(df.set_index('date'),
+                                    bt_data[bt_data["pair"] == 'POWR/BTC'],
+                                    "cum_profits")
+    assert "cum_profits" in cum_profits.columns
+    assert cum_profits.iloc[0]['cum_profits'] == 0
+    assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005

tests/data/test_converter.py

@@ -0,0 +1,148 @@
+# pragma pylint: disable=missing-docstring, C0103
+import logging
+
+from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data
+from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timeframe
+from freqtrade.tests.conftest import log_has
+
+
+def test_dataframe_correct_columns(result):
+    assert result.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume']
+
+
+def test_parse_ticker_dataframe(ticker_history_list, caplog):
+    columns = ['date', 'open', 'high', 'low', 'close', 'volume']
+
+    caplog.set_level(logging.DEBUG)
+    # Test file with BV data
+    dataframe = parse_ticker_dataframe(ticker_history_list, '5m',
+                                       pair="UNITTEST/BTC", fill_missing=True)
+    assert dataframe.columns.tolist() == columns
+    assert log_has('Parsing tickerlist to dataframe', caplog)
+
+
+def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
+    data = load_pair_history(datadir=testdatadir,
+                             ticker_interval='1m',
+                             refresh_pairs=False,
+                             pair='UNITTEST/BTC',
+                             fill_up_missing=False)
+    caplog.set_level(logging.DEBUG)
+    data2 = ohlcv_fill_up_missing_data(data, '1m', 'UNITTEST/BTC')
+    assert len(data2) > len(data)
+    # Column names should not change
+    assert (data.columns == data2.columns).all()
+
+    assert log_has(f"Missing data fillup for UNITTEST/BTC: before: "
+                   f"{len(data)} - after: {len(data2)}", caplog)
+
+    # Test fillup actually fixes invalid backtest data
+    min_date, max_date = get_timeframe({'UNITTEST/BTC': data})
+    assert validate_backtest_data(data, 'UNITTEST/BTC', min_date, max_date, 1)
+    assert not validate_backtest_data(data2, 'UNITTEST/BTC', min_date, max_date, 1)
+
+
+def test_ohlcv_fill_up_missing_data2(caplog):
+    ticker_interval = '5m'
+    ticks = [[
+            1511686200000,  # 8:50:00
+            8.794e-05,  # open
+            8.948e-05,  # high
+            8.794e-05,  # low
+            8.88e-05,  # close
+            2255,  # volume (in quote currency)
+        ],
+        [
+            1511686500000,  # 8:55:00
+            8.88e-05,
+            8.942e-05,
+            8.88e-05,
+            8.893e-05,
+            9911,
+        ],
+        [
+            1511687100000,  # 9:05:00
+            8.891e-05,
+            8.893e-05,
+            8.875e-05,
+            8.877e-05,
+            2251
+        ],
+        [
+            1511687400000,  # 9:10:00
+            8.877e-05,
+            8.883e-05,
+            8.895e-05,
+            8.817e-05,
+            123551
+    ]
+    ]
+
+    # Generate test-data without filling missing
+    data = parse_ticker_dataframe(ticks, ticker_interval, pair="UNITTEST/BTC", fill_missing=False)
+    assert len(data) == 3
+    caplog.set_level(logging.DEBUG)
+    data2 = ohlcv_fill_up_missing_data(data, ticker_interval, "UNITTEST/BTC")
+    assert len(data2) == 4
+    # 3rd candle has been filled
+    row = data2.loc[2, :]
+    assert row['volume'] == 0
+    # close shoult match close of previous candle
+    assert row['close'] == data.loc[1, 'close']
+    assert row['open'] == row['close']
+    assert row['high'] == row['close']
+    assert row['low'] == row['close']
+    # Column names should not change
+    assert (data.columns == data2.columns).all()
+
+    assert log_has(f"Missing data fillup for UNITTEST/BTC: before: "
+                   f"{len(data)} - after: {len(data2)}", caplog)
+
+
+def test_ohlcv_drop_incomplete(caplog):
+    ticker_interval = '1d'
+    ticks = [[
+            1559750400000,  # 2019-06-04
+            8.794e-05,  # open
+            8.948e-05,  # high
+            8.794e-05,  # low
+            8.88e-05,  # close
+            2255,  # volume (in quote currency)
+        ],
+        [
+            1559836800000,  # 2019-06-05
+            8.88e-05,
+            8.942e-05,
+            8.88e-05,
+            8.893e-05,
+            9911,
+        ],
+        [
+            1559923200000,  # 2019-06-06
+            8.891e-05,
+            8.893e-05,
+            8.875e-05,
+            8.877e-05,
+            2251
+        ],
+        [
+            1560009600000,  # 2019-06-07
+            8.877e-05,
+            8.883e-05,
+            8.895e-05,
+            8.817e-05,
+            123551
+     ]
+    ]
+    caplog.set_level(logging.DEBUG)
+    data = parse_ticker_dataframe(ticks, ticker_interval, pair="UNITTEST/BTC",
+                                  fill_missing=False, drop_incomplete=False)
+    assert len(data) == 4
+    assert not log_has("Dropping last candle", caplog)
+
+    # Drop last candle
+    data = parse_ticker_dataframe(ticks, ticker_interval, pair="UNITTEST/BTC",
+                                  fill_missing=False, drop_incomplete=True)
+    assert len(data) == 3
+
+    assert log_has("Dropping last candle", caplog)

tests/data/test_dataprovider.py

@@ -0,0 +1,123 @@
+from unittest.mock import MagicMock
+
+from pandas import DataFrame
+
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.state import RunMode
+from freqtrade.tests.conftest import get_patched_exchange
+
+
+def test_ohlcv(mocker, default_conf, ticker_history):
+    default_conf["runmode"] = RunMode.DRY_RUN
+    ticker_interval = default_conf["ticker_interval"]
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history
+
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.DRY_RUN
+    assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", ticker_interval))
+    assert isinstance(dp.ohlcv("UNITTEST/BTC", ticker_interval), DataFrame)
+    assert dp.ohlcv("UNITTEST/BTC", ticker_interval) is not ticker_history
+    assert dp.ohlcv("UNITTEST/BTC", ticker_interval, copy=False) is ticker_history
+    assert not dp.ohlcv("UNITTEST/BTC", ticker_interval).empty
+    assert dp.ohlcv("NONESENSE/AAA", ticker_interval).empty
+
+    # Test with and without parameter
+    assert dp.ohlcv("UNITTEST/BTC", ticker_interval).equals(dp.ohlcv("UNITTEST/BTC"))
+
+    default_conf["runmode"] = RunMode.LIVE
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.LIVE
+    assert isinstance(dp.ohlcv("UNITTEST/BTC", ticker_interval), DataFrame)
+
+    default_conf["runmode"] = RunMode.BACKTEST
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.BACKTEST
+    assert dp.ohlcv("UNITTEST/BTC", ticker_interval).empty
+
+
+def test_historic_ohlcv(mocker, default_conf, ticker_history):
+    historymock = MagicMock(return_value=ticker_history)
+    mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
+
+    dp = DataProvider(default_conf, None)
+    data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
+    assert isinstance(data, DataFrame)
+    assert historymock.call_count == 1
+    assert historymock.call_args_list[0][1]["refresh_pairs"] is False
+    assert historymock.call_args_list[0][1]["ticker_interval"] == "5m"
+
+
+def test_get_pair_dataframe(mocker, default_conf, ticker_history):
+    default_conf["runmode"] = RunMode.DRY_RUN
+    ticker_interval = default_conf["ticker_interval"]
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history
+
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.DRY_RUN
+    assert ticker_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval))
+    assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame)
+    assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval) is not ticker_history
+    assert not dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval).empty
+    assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty
+
+    # Test with and without parameter
+    assert dp.get_pair_dataframe("UNITTEST/BTC",
+                                 ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC"))
+
+    default_conf["runmode"] = RunMode.LIVE
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.LIVE
+    assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame)
+    assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty
+
+    historymock = MagicMock(return_value=ticker_history)
+    mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
+    default_conf["runmode"] = RunMode.BACKTEST
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.BACKTEST
+    assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame)
+    # assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty
+
+
+def test_available_pairs(mocker, default_conf, ticker_history):
+    exchange = get_patched_exchange(mocker, default_conf)
+    ticker_interval = default_conf["ticker_interval"]
+    exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history
+
+    dp = DataProvider(default_conf, exchange)
+    assert len(dp.available_pairs) == 2
+    assert dp.available_pairs == [
+        ("XRP/BTC", ticker_interval),
+        ("UNITTEST/BTC", ticker_interval),
+    ]
+
+
+def test_refresh(mocker, default_conf, ticker_history):
+    refresh_mock = MagicMock()
+    mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
+
+    exchange = get_patched_exchange(mocker, default_conf, id="binance")
+    ticker_interval = default_conf["ticker_interval"]
+    pairs = [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval)]
+
+    pairs_non_trad = [("ETH/USDT", ticker_interval), ("BTC/TUSD", "1h")]
+
+    dp = DataProvider(default_conf, exchange)
+    dp.refresh(pairs)
+
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args[0]) == 1
+    assert len(refresh_mock.call_args[0][0]) == len(pairs)
+    assert refresh_mock.call_args[0][0] == pairs
+
+    refresh_mock.reset_mock()
+    dp.refresh(pairs, pairs_non_trad)
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args[0]) == 1
+    assert len(refresh_mock.call_args[0][0]) == len(pairs) + len(pairs_non_trad)
+    assert refresh_mock.call_args[0][0] == pairs + pairs_non_trad

tests/data/test_history.py

@@ -0,0 +1,602 @@
+# pragma pylint: disable=missing-docstring, protected-access, C0103
+
+import json
+import os
+import uuid
+from pathlib import Path
+from shutil import copyfile
+from unittest.mock import MagicMock, PropertyMock
+
+import arrow
+import pytest
+from pandas import DataFrame
+
+from freqtrade import OperationalException
+from freqtrade.configuration import TimeRange
+from freqtrade.data import history
+from freqtrade.data.history import (download_pair_history,
+                                    load_cached_data_for_updating,
+                                    load_tickerdata_file,
+                                    refresh_backtest_ohlcv_data,
+                                    trim_tickerlist)
+from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.misc import file_dump_json
+from freqtrade.strategy.default_strategy import DefaultStrategy
+from freqtrade.tests.conftest import get_patched_exchange, log_has, log_has_re, patch_exchange
+
+# Change this if modifying UNITTEST/BTC testdatafile
+_BTC_UNITTEST_LENGTH = 13681
+
+
+def _backup_file(file: str, copy_file: bool = False) -> None:
+    """
+    Backup existing file to avoid deleting the user file
+    :param file: complete path to the file
+    :param touch_file: create an empty file in replacement
+    :return: None
+    """
+    file_swp = file + '.swp'
+    if os.path.isfile(file):
+        os.rename(file, file_swp)
+
+        if copy_file:
+            copyfile(file_swp, file)
+
+
+def _clean_test_file(file: str) -> None:
+    """
+    Backup existing file to avoid deleting the user file
+    :param file: complete path to the file
+    :return: None
+    """
+    file_swp = file + '.swp'
+    # 1. Delete file from the test
+    if os.path.isfile(file):
+        os.remove(file)
+
+    # 2. Rollback to the initial file
+    if os.path.isfile(file_swp):
+        os.rename(file_swp, file)
+
+
+def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> None:
+    ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=testdatadir)
+    assert isinstance(ld, DataFrame)
+    assert not log_has(
+        'Download history data for pair: "UNITTEST/BTC", interval: 30m '
+        'and store in None.', caplog
+    )
+
+
+def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> None:
+    ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='7m', datadir=testdatadir)
+    assert not isinstance(ld, DataFrame)
+    assert ld is None
+    assert log_has(
+        'No history data for pair: "UNITTEST/BTC", interval: 7m. '
+        'Use --refresh-pairs-cached option or `freqtrade download-data` '
+        'script to download the data', caplog
+    )
+
+
+def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> None:
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history)
+    file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json')
+    _backup_file(file, copy_file=True)
+    history.load_data(datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC'])
+    assert os.path.isfile(file) is True
+    assert not log_has(
+        'Download history data for pair: "UNITTEST/BTC", interval: 1m '
+        'and store in None.', caplog
+    )
+    _clean_test_file(file)
+
+
+def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
+                                      default_conf, testdatadir) -> None:
+    """
+    Test load_pair_history() with 1 min ticker
+    """
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list)
+    exchange = get_patched_exchange(mocker, default_conf)
+    file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
+
+    _backup_file(file)
+    # do not download a new pair if refresh_pairs isn't set
+    history.load_pair_history(datadir=testdatadir,
+                              ticker_interval='1m',
+                              refresh_pairs=False,
+                              pair='MEME/BTC')
+    assert os.path.isfile(file) is False
+    assert log_has(
+        'No history data for pair: "MEME/BTC", interval: 1m. '
+        'Use --refresh-pairs-cached option or `freqtrade download-data` '
+        'script to download the data', caplog
+    )
+
+    # download a new pair if refresh_pairs is set
+    history.load_pair_history(datadir=testdatadir,
+                              ticker_interval='1m',
+                              refresh_pairs=True,
+                              exchange=exchange,
+                              pair='MEME/BTC')
+    assert os.path.isfile(file) is True
+    assert log_has_re(
+        'Download history data for pair: "MEME/BTC", interval: 1m '
+        'and store in .*', caplog
+    )
+    with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'):
+        history.load_pair_history(datadir=testdatadir,
+                                  ticker_interval='1m',
+                                  refresh_pairs=True,
+                                  exchange=None,
+                                  pair='MEME/BTC')
+    _clean_test_file(file)
+
+
+def test_load_data_live(default_conf, mocker, caplog, testdatadir) -> None:
+    refresh_mock = MagicMock()
+    mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    history.load_data(datadir=testdatadir, ticker_interval='5m',
+                      pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
+                      live=True,
+                      exchange=exchange)
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args_list[0][0][0]) == 2
+    assert log_has('Live: Downloading data for all defined pairs ...', caplog)
+
+
+def test_load_data_live_noexchange(default_conf, mocker, caplog, testdatadir) -> None:
+
+    with pytest.raises(OperationalException,
+                       match=r'Exchange needs to be initialized when using live data.'):
+        history.load_data(datadir=testdatadir, ticker_interval='5m',
+                          pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
+                          exchange=None,
+                          live=True,
+                          )
+
+
+def test_testdata_path(testdatadir) -> None:
+    assert str(Path('freqtrade') / 'tests' / 'testdata') in str(testdatadir)
+
+
+def test_load_cached_data_for_updating(mocker) -> None:
+    datadir = Path(__file__).parent.parent.joinpath('testdata')
+
+    test_data = None
+    test_filename = datadir.joinpath('UNITTEST_BTC-1m.json')
+    with open(test_filename, "rt") as file:
+        test_data = json.load(file)
+
+    # change now time to test 'line' cases
+    # now = last cached item + 1 hour
+    now_ts = test_data[-1][0] / 1000 + 60 * 60
+    mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts))
+
+    # timeframe starts earlier than the cached data
+    # should fully update data
+    timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    assert data == []
+    assert start_ts == test_data[0][0] - 1000
+
+    # same with 'line' timeframe
+    num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m',
+                                                   TimeRange(None, 'line', 0, -num_lines))
+    assert data == []
+    assert start_ts < test_data[0][0] - 1
+
+    # timeframe starts in the center of the cached data
+    # should return the chached data w/o the last item
+    timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # same with 'line' timeframe
+    num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30
+    timerange = TimeRange(None, 'line', 0, -num_lines)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # timeframe starts after the chached data
+    # should return the chached data w/o the last item
+    timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # Try loading last 30 lines.
+    # Not supported by load_cached_data_for_updating, we always need to get the full data.
+    num_lines = 30
+    timerange = TimeRange(None, 'line', 0, -num_lines)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # no timeframe is set
+    # should return the chached data w/o the last item
+    num_lines = 30
+    timerange = TimeRange(None, 'line', 0, -num_lines)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # no datafile exist
+    # should return timestamp start time
+    timerange = TimeRange('date', None, now_ts - 10000, 0)
+    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
+    assert data == []
+    assert start_ts == (now_ts - 10000) * 1000
+
+    # same with 'line' timeframe
+    num_lines = 30
+    timerange = TimeRange(None, 'line', 0, -num_lines)
+    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
+    assert data == []
+    assert start_ts == (now_ts - num_lines * 60) * 1000
+
+    # no datafile exist, no timeframe is set
+    # should return an empty array and None
+    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None)
+    assert data == []
+    assert start_ts is None
+
+
+def test_download_pair_history(ticker_history_list, mocker, default_conf, testdatadir) -> None:
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list)
+    exchange = get_patched_exchange(mocker, default_conf)
+    file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
+    file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
+    file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json')
+    file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-5m.json')
+
+    _backup_file(file1_1)
+    _backup_file(file1_5)
+    _backup_file(file2_1)
+    _backup_file(file2_5)
+
+    assert os.path.isfile(file1_1) is False
+    assert os.path.isfile(file2_1) is False
+
+    assert download_pair_history(datadir=testdatadir, exchange=exchange,
+                                 pair='MEME/BTC',
+                                 ticker_interval='1m')
+    assert download_pair_history(datadir=testdatadir, exchange=exchange,
+                                 pair='CFI/BTC',
+                                 ticker_interval='1m')
+    assert not exchange._pairs_last_refresh_time
+    assert os.path.isfile(file1_1) is True
+    assert os.path.isfile(file2_1) is True
+
+    # clean files freshly downloaded
+    _clean_test_file(file1_1)
+    _clean_test_file(file2_1)
+
+    assert os.path.isfile(file1_5) is False
+    assert os.path.isfile(file2_5) is False
+
+    assert download_pair_history(datadir=testdatadir, exchange=exchange,
+                                 pair='MEME/BTC',
+                                 ticker_interval='5m')
+    assert download_pair_history(datadir=testdatadir, exchange=exchange,
+                                 pair='CFI/BTC',
+                                 ticker_interval='5m')
+    assert not exchange._pairs_last_refresh_time
+    assert os.path.isfile(file1_5) is True
+    assert os.path.isfile(file2_5) is True
+
+    # clean files freshly downloaded
+    _clean_test_file(file1_5)
+    _clean_test_file(file2_5)
+
+
+def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
+    tick = [
+        [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839],
+        [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199]
+    ]
+    json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
+    exchange = get_patched_exchange(mocker, default_conf)
+    download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", ticker_interval='1m')
+    download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", ticker_interval='3m')
+    assert json_dump_mock.call_count == 2
+
+
+def test_download_backtesting_data_exception(ticker_history, mocker, caplog,
+                                             default_conf, testdatadir) -> None:
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv',
+                 side_effect=Exception('File Error'))
+
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
+    file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
+    _backup_file(file1_1)
+    _backup_file(file1_5)
+
+    assert not download_pair_history(datadir=testdatadir, exchange=exchange,
+                                     pair='MEME/BTC',
+                                     ticker_interval='1m')
+    # clean files freshly downloaded
+    _clean_test_file(file1_1)
+    _clean_test_file(file1_5)
+    assert log_has(
+        'Failed to download history data for pair: "MEME/BTC", interval: 1m. '
+        'Error: File Error', caplog
+    )
+
+
+def test_load_tickerdata_file(testdatadir) -> None:
+    # 7 does not exist in either format.
+    assert not load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '7m')
+    # 1 exists only as a .json
+    tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m')
+    assert _BTC_UNITTEST_LENGTH == len(tickerdata)
+    # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json
+    tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '8m')
+    assert _BTC_UNITTEST_LENGTH == len(tickerdata)
+
+
+def test_load_partial_missing(testdatadir, caplog) -> None:
+    # Make sure we start fresh - test missing data at start
+    start = arrow.get('2018-01-01T00:00:00')
+    end = arrow.get('2018-01-11T00:00:00')
+    tickerdata = history.load_data(testdatadir, '5m', ['UNITTEST/BTC'],
+                                   refresh_pairs=False,
+                                   timerange=TimeRange('date', 'date',
+                                                       start.timestamp, end.timestamp))
+    # timedifference in 5 minutes
+    td = ((end - start).total_seconds() // 60 // 5) + 1
+    assert td != len(tickerdata['UNITTEST/BTC'])
+    start_real = tickerdata['UNITTEST/BTC'].iloc[0, 0]
+    assert log_has(f'Missing data at start for pair '
+                   f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}',
+                   caplog)
+    # Make sure we start fresh - test missing data at end
+    caplog.clear()
+    start = arrow.get('2018-01-10T00:00:00')
+    end = arrow.get('2018-02-20T00:00:00')
+    tickerdata = history.load_data(datadir=testdatadir, ticker_interval='5m',
+                                   pairs=['UNITTEST/BTC'], refresh_pairs=False,
+                                   timerange=TimeRange('date', 'date',
+                                                       start.timestamp, end.timestamp))
+    # timedifference in 5 minutes
+    td = ((end - start).total_seconds() // 60 // 5) + 1
+    assert td != len(tickerdata['UNITTEST/BTC'])
+    # Shift endtime with +5 - as last candle is dropped (partial candle)
+    end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5)
+    assert log_has(f'Missing data at end for pair '
+                   f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}',
+                   caplog)
+
+
+def test_init(default_conf, mocker) -> None:
+    exchange = get_patched_exchange(mocker, default_conf)
+    assert {} == history.load_data(
+        datadir='',
+        exchange=exchange,
+        pairs=[],
+        refresh_pairs=True,
+        ticker_interval=default_conf['ticker_interval']
+    )
+
+
+def test_trim_tickerlist() -> None:
+    file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json')
+    with open(file) as data_file:
+        ticker_list = json.load(data_file)
+    ticker_list_len = len(ticker_list)
+
+    # Test the pattern ^(-\d+)$
+    # This pattern uses the latest N elements
+    timerange = TimeRange(None, 'line', 0, -5)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is not ticker[0]  # The first element should be different
+    assert ticker_list[-1] is ticker[-1]  # The last element must be the same
+
+    # Test the pattern ^(\d+)-$
+    # This pattern keep X element from the end
+    timerange = TimeRange('line', None, 5, 0)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is ticker[0]  # The first element must be the same
+    assert ticker_list[-1] is not ticker[-1]  # The last element should be different
+
+    # Test the pattern ^(\d+)-(\d+)$
+    # This pattern extract a window
+    timerange = TimeRange('index', 'index', 5, 10)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is not ticker[0]  # The first element should be different
+    assert ticker_list[5] is ticker[0]  # The list starts at the index 5
+    assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
+
+    # Test the pattern ^(\d{8})-(\d{8})$
+    # This pattern extract a window between the dates
+    timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is not ticker[0]  # The first element should be different
+    assert ticker_list[5] is ticker[0]  # The list starts at the index 5
+    assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
+
+    # Test the pattern ^-(\d{8})$
+    # This pattern extracts elements from the start to the date
+    timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 10
+    assert ticker_list[0] is ticker[0]  # The start of the list is included
+    assert ticker_list[9] is ticker[-1]  # The element 10 is not included
+
+    # Test the pattern ^(\d{8})-$
+    # This pattern extracts elements from the date to now
+    timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == ticker_list_len - 10
+    assert ticker_list[10] is ticker[0]  # The first element is element #10
+    assert ticker_list[-1] is ticker[-1]  # The last element is the same
+
+    # Test a wrong pattern
+    # This pattern must return the list unchanged
+    timerange = TimeRange(None, None, None, 5)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_list_len == ticker_len
+
+    # Test invalid timerange (start after stop)
+    timerange = TimeRange('index', 'index', 10, 5)
+    with pytest.raises(ValueError, match=r'The timerange .* is incorrect'):
+        trim_tickerlist(ticker_list, timerange)
+
+    assert ticker_list_len == ticker_len
+
+    # passing empty list
+    timerange = TimeRange(None, None, None, 5)
+    ticker = trim_tickerlist([], timerange)
+    assert 0 == len(ticker)
+    assert not ticker
+
+
+def test_file_dump_json_tofile() -> None:
+    file = os.path.join(os.path.dirname(__file__), '..', 'testdata',
+                        'test_{id}.json'.format(id=str(uuid.uuid4())))
+    data = {'bar': 'foo'}
+
+    # check the file we will create does not exist
+    assert os.path.isfile(file) is False
+
+    # Create the Json file
+    file_dump_json(file, data)
+
+    # Check the file was create
+    assert os.path.isfile(file) is True
+
+    # Open the Json file created and test the data is in it
+    with open(file) as data_file:
+        json_from_file = json.load(data_file)
+
+    assert 'bar' in json_from_file
+    assert json_from_file['bar'] == 'foo'
+
+    # Remove the file
+    _clean_test_file(file)
+
+
+def test_get_timeframe(default_conf, mocker, testdatadir) -> None:
+    patch_exchange(mocker)
+    strategy = DefaultStrategy(default_conf)
+
+    data = strategy.tickerdata_to_dataframe(
+        history.load_data(
+            datadir=testdatadir,
+            ticker_interval='1m',
+            pairs=['UNITTEST/BTC']
+        )
+    )
+    min_date, max_date = history.get_timeframe(data)
+    assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
+    assert max_date.isoformat() == '2017-11-14T22:58:00+00:00'
+
+
+def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) -> None:
+    patch_exchange(mocker)
+    strategy = DefaultStrategy(default_conf)
+
+    data = strategy.tickerdata_to_dataframe(
+        history.load_data(
+            datadir=testdatadir,
+            ticker_interval='1m',
+            pairs=['UNITTEST/BTC'],
+            fill_up_missing=False
+        )
+    )
+    min_date, max_date = history.get_timeframe(data)
+    caplog.clear()
+    assert history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
+                                          min_date, max_date, timeframe_to_minutes('1m'))
+    assert len(caplog.record_tuples) == 1
+    assert log_has(
+        "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values",
+        caplog)
+
+
+def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> None:
+    patch_exchange(mocker)
+    strategy = DefaultStrategy(default_conf)
+
+    timerange = TimeRange('index', 'index', 200, 250)
+    data = strategy.tickerdata_to_dataframe(
+        history.load_data(
+            datadir=testdatadir,
+            ticker_interval='5m',
+            pairs=['UNITTEST/BTC'],
+            timerange=timerange
+        )
+    )
+
+    min_date, max_date = history.get_timeframe(data)
+    caplog.clear()
+    assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
+                                              min_date, max_date, timeframe_to_minutes('5m'))
+    assert len(caplog.record_tuples) == 0
+
+
+def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir):
+    dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
+    )
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    mocker.patch.object(Path, "unlink", MagicMock())
+
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
+                                timeframes=["1m", "5m"], dl_path=testdatadir,
+                                timerange=timerange, erase=True
+                                )
+
+    assert dl_mock.call_count == 4
+    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
+
+    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
+
+
+def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
+    dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
+                                             timeframes=["1m", "5m"],
+                                             dl_path=testdatadir,
+                                             timerange=timerange, erase=False
+                                             )
+
+    assert dl_mock.call_count == 0
+    assert "ETH/BTC" in unav_pairs
+    assert "XRP/BTC" in unav_pairs
+    assert log_has("Skipping pair ETH/BTC...", caplog)