diff --git a/freqtrade/data/datahandlers/__init__.py b/freqtrade/data/datahandlers/__init__.py deleted file mode 100644 index a21dd832d..000000000 --- a/freqtrade/data/datahandlers/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Type -from pathlib import Path -from .idatahandler import IDataHandler - - -def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: - """ - Get datahandler class. - Could be done using Resolvers, but since this may be called often and resolvers - are rather expensive, doing this directly should improve performance. - :param datatype: datatype to use. - :return: Datahandler class - """ - - if datatype == 'json': - from .jsondatahandler import JsonDataHandler - return JsonDataHandler - elif datatype == 'jsongz': - from .jsondatahandler import JsonGzDataHandler - return JsonGzDataHandler - else: - raise ValueError(f"No datahandler for datatype {datatype} available.") - - -def get_datahandler(datadir: Path, data_format: str = None, - data_handler: IDataHandler = None) -> IDataHandler: - """ - :param datadir: Folder to save data - :data_format: dataformat to use - :data_handler: returns this datahandler if it exists or initializes a new one - """ - - if not data_handler: - HandlerClass = get_datahandlerclass(data_format or 'json') - data_handler = HandlerClass(datadir) - return data_handler diff --git a/freqtrade/data/history/__init__.py b/freqtrade/data/history/__init__.py new file mode 100644 index 000000000..572c063fc --- /dev/null +++ b/freqtrade/data/history/__init__.py @@ -0,0 +1,14 @@ +""" +Handle historic data (ohlcv). + +Includes: +* load data for a pair (or a list of pairs) from disk +* download data from exchange and store to disk +""" + +from .history_utils import (convert_trades_to_ohlcv, # noqa: F401 + get_timerange, load_data, load_pair_history, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, + validate_backtest_data) +from .idatahandler import get_datahandler, get_datahandlerclass # noqa: F401 diff --git a/freqtrade/data/history.py b/freqtrade/data/history/history_utils.py similarity index 98% rename from freqtrade/data/history.py rename to freqtrade/data/history/history_utils.py index c06b14b4a..42f1c9be7 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history/history_utils.py @@ -1,11 +1,3 @@ -""" -Handle historic data (ohlcv). - -Includes: -* load data for a pair (or a list of pairs) from disk -* download data from exchange and store to disk -""" - import logging import operator from datetime import datetime, timezone @@ -19,8 +11,7 @@ from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.data.datahandlers import get_datahandler -from freqtrade.data.datahandlers.idatahandler import IDataHandler +from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/history/idatahandler.py similarity index 76% rename from freqtrade/data/datahandlers/idatahandler.py rename to freqtrade/data/history/idatahandler.py index b80b8cfcc..cee43dcef 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -6,13 +6,14 @@ It's subclasses handle and storing data from disk. import logging from abc import ABC, abstractclassmethod, abstractmethod from copy import deepcopy -from pathlib import Path -from typing import Dict, List, Optional from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Optional, Type + from pandas import DataFrame -from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from freqtrade.configuration import TimeRange +from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from freqtrade.exchange import timeframe_to_seconds logger = logging.getLogger(__name__) @@ -89,3 +90,36 @@ class IDataHandler(ABC): if pairdata.iloc[-1]['date'] < stop: logger.warning(f"Missing data at end for pair {pair}, " f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}") + + +def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: + """ + Get datahandler class. + Could be done using Resolvers, but since this may be called often and resolvers + are rather expensive, doing this directly should improve performance. + :param datatype: datatype to use. + :return: Datahandler class + """ + + if datatype == 'json': + from .jsondatahandler import JsonDataHandler + return JsonDataHandler + elif datatype == 'jsongz': + from .jsondatahandler import JsonGzDataHandler + return JsonGzDataHandler + else: + raise ValueError(f"No datahandler for datatype {datatype} available.") + + +def get_datahandler(datadir: Path, data_format: str = None, + data_handler: IDataHandler = None) -> IDataHandler: + """ + :param datadir: Folder to save data + :data_format: dataformat to use + :data_handler: returns this datahandler if it exists or initializes a new one + """ + + if not data_handler: + HandlerClass = get_datahandlerclass(data_format or 'json') + data_handler = HandlerClass(datadir) + return data_handler diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py similarity index 100% rename from freqtrade/data/datahandlers/jsondatahandler.py rename to freqtrade/data/history/jsondatahandler.py diff --git a/freqtrade/utils.py b/freqtrade/utils.py index f5865f1db..1bb0f611a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -17,8 +17,8 @@ from freqtrade.configuration import (Configuration, TimeRange, from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY -from freqtrade.data.datahandlers import get_datahandlerclass from freqtrade.data.history import (convert_trades_to_ohlcv, + get_datahandlerclass, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exchange import (available_exchanges, ccxt_exchanges, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 726b6d3a7..f84e819ad 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -13,18 +13,15 @@ from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass -from freqtrade.data.datahandlers.idatahandler import IDataHandler -from freqtrade.data.datahandlers.jsondatahandler import (JsonDataHandler, - JsonGzDataHandler) -from freqtrade.data.history import (_download_pair_history, - _download_trades_history, - _load_cached_data_for_updating, - convert_trades_to_ohlcv, get_timerange, - load_data, load_pair_history, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data, refresh_data, - validate_backtest_data) +from freqtrade.data.history import get_datahandler, get_datahandlerclass +from freqtrade.data.history.history_utils import ( + _download_pair_history, _download_trades_history, + _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, + load_data, load_pair_history, refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, validate_backtest_data) +from freqtrade.data.history.idatahandler import IDataHandler +from freqtrade.data.history.jsondatahandler import (JsonDataHandler, + JsonGzDataHandler) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy @@ -100,7 +97,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: ltfmock = mocker.patch( - 'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler._ohlcv_load', + 'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load', MagicMock(return_value=DataFrame())) timerange = TimeRange('date', None, 1510639620, 0) load_pair_history(pair='UNITTEST/BTC', timeframe='1m', @@ -271,7 +268,7 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch( - 'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler.ohlcv_store', + 'freqtrade.data.history.jsondatahandler.JsonDataHandler.ohlcv_store', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) @@ -444,7 +441,8 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', + MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -465,7 +463,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch( @@ -485,7 +483,8 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_trades_history', + MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 447f0052b..427e6c422 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -460,7 +460,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame())) + mocker.patch('freqtrade.data.history.history_utils.load_pair_history', + MagicMock(return_value=pd.DataFrame())) mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker)