diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4c5c0521f..f09fe3d6a 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -8,7 +8,6 @@ Includes: import logging import operator -from copy import deepcopy from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -18,8 +17,9 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange -from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds +from freqtrade.data.converter import trades_to_ohlcv +from freqtrade.exchange import Exchange, timeframe_to_minutes +from .datahandlers import get_datahandlerclass logger = logging.getLogger(__name__) @@ -126,11 +126,12 @@ def _validate_pairdata(pair, pairdata, timerange: TimeRange): def load_pair_history(pair: str, timeframe: str, - datadir: Path, + datadir: Path, *, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, + data_format: str = 'json' ) -> DataFrame: """ Load cached ticker history for the given pair. @@ -142,26 +143,18 @@ def load_pair_history(pair: str, :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period + :param data_format: Format of the data :return: DataFrame with ohlcv data, or empty DataFrame """ - timerange_startup = deepcopy(timerange) - if startup_candles > 0 and timerange_startup: - timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) - - pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) - - if pairdata: - if timerange_startup: - _validate_pairdata(pair, pairdata, timerange_startup) - return parse_ticker_dataframe(pairdata, timeframe, pair=pair, - fill_missing=fill_up_missing, - drop_incomplete=drop_incomplete) - else: - logger.warning( - f'No history data for pair: "{pair}", timeframe: {timeframe}. ' - 'Use `freqtrade download-data` to download the data' - ) - return DataFrame() + HandlerClass = get_datahandlerclass(data_format) + loader = HandlerClass(datadir) + return loader.ohlcv_load(pair=pair, + timeframe=timeframe, + timerange=timerange, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete, + startup_candles=startup_candles, + ) def load_data(datadir: Path, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 7b3143db9..f13e386f8 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -96,8 +96,9 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: - ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file', - MagicMock(return_value=None)) + ltfmock = mocker.patch( + 'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler._ohlcv_load', + MagicMock(return_value=DataFrame())) timerange = TimeRange('date', None, 1510639620, 0) load_pair_history(pair='UNITTEST/BTC', timeframe='1m', datadir=testdatadir, timerange=timerange, @@ -361,8 +362,8 @@ def test_load_partial_missing(testdatadir, caplog) -> None: # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(tickerdata['UNITTEST/BTC']) - # Shift endtime with +5 - as last candle is dropped (partial candle) - end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5) + # This validation happens now after parsing to pandas. + end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]) assert log_has(f'Missing data at end for pair ' f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog)