adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rename datahandler module to history module

Browse Source 
Also move previous history.py into this module - so everything is
bundled
This commit is contained in:
Matthias 2019-12-28 09:59:47 +01:00
parent b37b5c3d90
commit 6860491189
8 changed files with 71 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
freqtrade/data/datahandlers/__init__.py
View File

@ -1,36 +0,0 @@
from typing import Type
from pathlib import Path
from .idatahandler import IDataHandler
def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
"""
Get datahandler class.
Could be done using Resolvers, but since this may be called often and resolvers
are rather expensive, doing this directly should improve performance.
:param datatype: datatype to use.
:return: Datahandler class
"""
if datatype == 'json':
from .jsondatahandler import JsonDataHandler
return JsonDataHandler
elif datatype == 'jsongz':
from .jsondatahandler import JsonGzDataHandler
return JsonGzDataHandler
else:
raise ValueError(f"No datahandler for datatype {datatype} available.")
def get_datahandler(datadir: Path, data_format: str = None,
data_handler: IDataHandler = None) -> IDataHandler:
"""
:param datadir: Folder to save data
:data_format: dataformat to use
:data_handler: returns this datahandler if it exists or initializes a new one
"""
if not data_handler:
HandlerClass = get_datahandlerclass(data_format or 'json')
data_handler = HandlerClass(datadir)
return data_handler

14
freqtrade/data/history/__init__.py Normal file
View File

@ -0,0 +1,14 @@
"""
Handle historic data (ohlcv).
Includes:
* load data for a pair (or a list of pairs) from disk
* download data from exchange and store to disk
"""
from .history_utils import (convert_trades_to_ohlcv, # noqa: F401
get_timerange, load_data, load_pair_history,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data, refresh_data,
validate_backtest_data)
from .idatahandler import get_datahandler, get_datahandlerclass # noqa: F401

11
freqtrade/data/history.py → freqtrade/data/history/history_utils.py
View File

@ -1,11 +1,3 @@
"""
Handle historic data (ohlcv).
Includes:
* load data for a pair (or a list of pairs) from disk
* download data from exchange and store to disk
"""
import logging
import operator
from datetime import datetime, timezone
@ -19,8 +11,7 @@ from freqtrade import OperationalException
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
from freqtrade.data.datahandlers import get_datahandler
from freqtrade.data.datahandlers.idatahandler import IDataHandler
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)

40
freqtrade/data/datahandlers/idatahandler.py → freqtrade/data/history/idatahandler.py
View File

@ -6,13 +6,14 @@ It's subclasses handle and storing data from disk.
import logging
from abc import ABC, abstractclassmethod, abstractmethod
from copy import deepcopy
from pathlib import Path
from typing import Dict, List, Optional
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional, Type
from pandas import DataFrame
from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
from freqtrade.configuration import TimeRange
from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
from freqtrade.exchange import timeframe_to_seconds
logger = logging.getLogger(__name__)
@ -89,3 +90,36 @@ class IDataHandler(ABC):
if pairdata.iloc[-1]['date'] < stop:
logger.warning(f"Missing data at end for pair {pair}, "
f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")
def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
"""
Get datahandler class.
Could be done using Resolvers, but since this may be called often and resolvers
are rather expensive, doing this directly should improve performance.
:param datatype: datatype to use.
:return: Datahandler class
"""
if datatype == 'json':
from .jsondatahandler import JsonDataHandler
return JsonDataHandler
elif datatype == 'jsongz':
from .jsondatahandler import JsonGzDataHandler
return JsonGzDataHandler
else:
raise ValueError(f"No datahandler for datatype {datatype} available.")
def get_datahandler(datadir: Path, data_format: str = None,
data_handler: IDataHandler = None) -> IDataHandler:
"""
:param datadir: Folder to save data
:data_format: dataformat to use
:data_handler: returns this datahandler if it exists or initializes a new one
"""
if not data_handler:
HandlerClass = get_datahandlerclass(data_format or 'json')
data_handler = HandlerClass(datadir)
return data_handler

0
freqtrade/data/datahandlers/jsondatahandler.py → freqtrade/data/history/jsondatahandler.py
View File

2
freqtrade/utils.py
View File

@ -17,8 +17,8 @@ from freqtrade.configuration import (Configuration, TimeRange,
from freqtrade.configuration.directory_operations import (copy_sample_files,
create_userdata_dir)
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
from freqtrade.data.datahandlers import get_datahandlerclass
from freqtrade.data.history import (convert_trades_to_ohlcv,
get_datahandlerclass,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data)
from freqtrade.exchange import (available_exchanges, ccxt_exchanges,

33
tests/data/test_history.py
View File

@ -13,18 +13,15 @@ from pandas.testing import assert_frame_equal
from freqtrade.configuration import TimeRange
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass
from freqtrade.data.datahandlers.idatahandler import IDataHandler
from freqtrade.data.datahandlers.jsondatahandler import (JsonDataHandler,
JsonGzDataHandler)
from freqtrade.data.history import (_download_pair_history,
_download_trades_history,
_load_cached_data_for_updating,
convert_trades_to_ohlcv, get_timerange,
load_data, load_pair_history,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data, refresh_data,
validate_backtest_data)
from freqtrade.data.history import get_datahandler, get_datahandlerclass
from freqtrade.data.history.history_utils import (
_download_pair_history, _download_trades_history,
_load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange,
load_data, load_pair_history, refresh_backtest_ohlcv_data,
refresh_backtest_trades_data, refresh_data, validate_backtest_data)
from freqtrade.data.history.idatahandler import IDataHandler
from freqtrade.data.history.jsondatahandler import (JsonDataHandler,
JsonGzDataHandler)
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json
from freqtrade.strategy.default_strategy import DefaultStrategy
@ -100,7 +97,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N
def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None:
ltfmock = mocker.patch(
'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler._ohlcv_load',
'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load',
MagicMock(return_value=DataFrame()))
timerange = TimeRange('date', None, 1510639620, 0)
load_pair_history(pair='UNITTEST/BTC', timeframe='1m',
@ -271,7 +268,7 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
[1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199]
]
json_dump_mock = mocker.patch(
'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler.ohlcv_store',
'freqtrade.data.history.jsondatahandler.JsonDataHandler.ohlcv_store',
return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf)
@ -444,7 +441,8 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No
def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history',
MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
@ -465,7 +463,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock())
ex = get_patched_exchange(mocker, default_conf)
mocker.patch(
@ -485,7 +483,8 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock())
dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_trades_history',
MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)

3
tests/optimize/test_backtesting.py
View File

@ -460,7 +460,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
def get_timerange(input1):
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame()))
mocker.patch('freqtrade.data.history.history_utils.load_pair_history',
MagicMock(return_value=pd.DataFrame()))
mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
patch_exchange(mocker)