Rename datahandler module to history module

Also move previous history.py into this module - so everything is
bundled

freqtrade/data/datahandlers/__init__.py

@@ -1,36 +0,0 @@
-from typing import Type
-from pathlib import Path
-from .idatahandler import IDataHandler
-
-
-def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
-    """
-    Get datahandler class.
-    Could be done using Resolvers, but since this may be called often and resolvers
-    are rather expensive, doing this directly should improve performance.
-    :param datatype: datatype to use.
-    :return: Datahandler class
-    """
-
-    if datatype == 'json':
-        from .jsondatahandler import JsonDataHandler
-        return JsonDataHandler
-    elif datatype == 'jsongz':
-        from .jsondatahandler import JsonGzDataHandler
-        return JsonGzDataHandler
-    else:
-        raise ValueError(f"No datahandler for datatype {datatype} available.")
-
-
-def get_datahandler(datadir: Path, data_format: str = None,
-                    data_handler: IDataHandler = None) -> IDataHandler:
-    """
-    :param datadir: Folder to save data
-    :data_format: dataformat to use
-    :data_handler: returns this datahandler if it exists or initializes a new one
-    """
-
-    if not data_handler:
-        HandlerClass = get_datahandlerclass(data_format or 'json')
-        data_handler = HandlerClass(datadir)
-    return data_handler

freqtrade/data/history/__init__.py

@@ -0,0 +1,14 @@
+"""
+Handle historic data (ohlcv).
+
+Includes:
+* load data for a pair (or a list of pairs) from disk
+* download data from exchange and store to disk
+"""
+
+from .history_utils import (convert_trades_to_ohlcv,  # noqa: F401
+                            get_timerange, load_data, load_pair_history,
+                            refresh_backtest_ohlcv_data,
+                            refresh_backtest_trades_data, refresh_data,
+                            validate_backtest_data)
+from .idatahandler import get_datahandler, get_datahandlerclass  # noqa: F401

freqtrade/data/history/history_utils.py

@@ -1,11 +1,3 @@
-"""
-Handle historic data (ohlcv).
-
-Includes:
-* load data for a pair (or a list of pairs) from disk
-* download data from exchange and store to disk
-"""
-
 import logging
 import operator
 from datetime import datetime, timezone
@@ -19,8 +11,7 @@ from freqtrade import OperationalException
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
 from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
-from freqtrade.data.datahandlers import get_datahandler
+from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
-from freqtrade.data.datahandlers.idatahandler import IDataHandler
 from freqtrade.exchange import Exchange
 
 logger = logging.getLogger(__name__)

freqtrade/data/history/idatahandler.py

@@ -6,13 +6,14 @@ It's subclasses handle and storing data from disk.
 import logging
 from abc import ABC, abstractclassmethod, abstractmethod
 from copy import deepcopy
-from pathlib import Path
-from typing import Dict, List, Optional
 from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Optional, Type
+
 from pandas import DataFrame
-from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
 
 from freqtrade.configuration import TimeRange
+from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
 from freqtrade.exchange import timeframe_to_seconds
 
 logger = logging.getLogger(__name__)
@@ -89,3 +90,36 @@ class IDataHandler(ABC):
             if pairdata.iloc[-1]['date'] < stop:
                 logger.warning(f"Missing data at end for pair {pair}, "
                                f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")
+
+
+def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
+    """
+    Get datahandler class.
+    Could be done using Resolvers, but since this may be called often and resolvers
+    are rather expensive, doing this directly should improve performance.
+    :param datatype: datatype to use.
+    :return: Datahandler class
+    """
+
+    if datatype == 'json':
+        from .jsondatahandler import JsonDataHandler
+        return JsonDataHandler
+    elif datatype == 'jsongz':
+        from .jsondatahandler import JsonGzDataHandler
+        return JsonGzDataHandler
+    else:
+        raise ValueError(f"No datahandler for datatype {datatype} available.")
+
+
+def get_datahandler(datadir: Path, data_format: str = None,
+                    data_handler: IDataHandler = None) -> IDataHandler:
+    """
+    :param datadir: Folder to save data
+    :data_format: dataformat to use
+    :data_handler: returns this datahandler if it exists or initializes a new one
+    """
+
+    if not data_handler:
+        HandlerClass = get_datahandlerclass(data_format or 'json')
+        data_handler = HandlerClass(datadir)
+    return data_handler

freqtrade/utils.py

@@ -17,8 +17,8 @@ from freqtrade.configuration import (Configuration, TimeRange,
 from freqtrade.configuration.directory_operations import (copy_sample_files,
                                                           create_userdata_dir)
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
-from freqtrade.data.datahandlers import get_datahandlerclass
 from freqtrade.data.history import (convert_trades_to_ohlcv,
+                                    get_datahandlerclass,
                                     refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
 from freqtrade.exchange import (available_exchanges, ccxt_exchanges,

tests/data/test_history.py

@@ -13,18 +13,15 @@ from pandas.testing import assert_frame_equal
 
 from freqtrade.configuration import TimeRange
 from freqtrade.data.converter import parse_ticker_dataframe
-from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass
+from freqtrade.data.history import get_datahandler, get_datahandlerclass
-from freqtrade.data.datahandlers.idatahandler import IDataHandler
+from freqtrade.data.history.history_utils import (
-from freqtrade.data.datahandlers.jsondatahandler import (JsonDataHandler,
+    _download_pair_history, _download_trades_history,
+    _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange,
+    load_data, load_pair_history, refresh_backtest_ohlcv_data,
+    refresh_backtest_trades_data, refresh_data, validate_backtest_data)
+from freqtrade.data.history.idatahandler import IDataHandler
+from freqtrade.data.history.jsondatahandler import (JsonDataHandler,
                                                     JsonGzDataHandler)
-from freqtrade.data.history import (_download_pair_history,
-                                    _download_trades_history,
-                                    _load_cached_data_for_updating,
-                                    convert_trades_to_ohlcv, get_timerange,
-                                    load_data, load_pair_history,
-                                    refresh_backtest_ohlcv_data,
-                                    refresh_backtest_trades_data, refresh_data,
-                                    validate_backtest_data)
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.strategy.default_strategy import DefaultStrategy
@@ -100,7 +97,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N
 
 def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None:
     ltfmock = mocker.patch(
-        'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler._ohlcv_load',
+        'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load',
         MagicMock(return_value=DataFrame()))
     timerange = TimeRange('date', None, 1510639620, 0)
     load_pair_history(pair='UNITTEST/BTC', timeframe='1m',
@@ -271,7 +268,7 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
         [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199]
     ]
     json_dump_mock = mocker.patch(
-        'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler.ohlcv_store',
+        'freqtrade.data.history.jsondatahandler.JsonDataHandler.ohlcv_store',
         return_value=None)
     mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
     exchange = get_patched_exchange(mocker, default_conf)
@@ -444,7 +441,8 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No
 
 
 def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir):
-    dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
+    dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history',
+                           MagicMock())
     mocker.patch(
         'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
     )
@@ -465,7 +463,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
 
 
 def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
-    dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
+    dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock())
 
     ex = get_patched_exchange(mocker, default_conf)
     mocker.patch(
@@ -485,7 +483,8 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
 
 
 def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
-    dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock())
+    dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_trades_history',
+                           MagicMock())
     mocker.patch(
         'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
     )

tests/optimize/test_backtesting.py

@@ -460,7 +460,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
     def get_timerange(input1):
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
-    mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame()))
+    mocker.patch('freqtrade.data.history.history_utils.load_pair_history',
+                 MagicMock(return_value=pd.DataFrame()))
     mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)