stable/freqtrade/data/history/hdf5datahandler.py

216 lines
8.1 KiB
Python
Raw Normal View History

import logging
import re
from pathlib import Path
from typing import List, Optional
import numpy as np
import pandas as pd
from freqtrade.configuration import TimeRange
2020-09-28 17:39:41 +00:00
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS,
ListPairsWithTimeframes, TradeList)
2022-03-03 06:06:13 +00:00
from freqtrade.enums import CandleType, TradingMode
from .idatahandler import IDataHandler
2020-09-28 17:39:41 +00:00
logger = logging.getLogger(__name__)
2020-07-24 17:23:37 +00:00
class HDF5DataHandler(IDataHandler):
_columns = DEFAULT_DATAFRAME_COLUMNS
2020-07-24 15:30:16 +00:00
@classmethod
2022-03-03 06:06:13 +00:00
def ohlcv_get_available_data(
cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes:
2020-07-24 15:30:16 +00:00
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
2020-07-24 15:30:16 +00:00
:return: List of Tuples of (pair, timeframe)
"""
2022-03-03 06:06:13 +00:00
if trading_mode == TradingMode.FUTURES:
datadir = datadir.joinpath('futures')
_tmp = [
re.search(
2021-11-28 13:33:46 +00:00
cls._OHLCV_REGEX, p.name
) for p in datadir.glob("*.h5")
]
2021-12-03 12:04:31 +00:00
return [
(
cls.rebuild_pair_from_filename(match[1]),
match[2],
CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1]
2020-07-24 15:30:16 +00:00
@classmethod
2021-12-07 19:30:58 +00:00
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]:
"""
Returns a list of all pairs with ohlcv data available in this datadir
for the specified timeframe
:param datadir: Directory to search for ohlcv files
:param timeframe: Timeframe to search pairs for
2021-12-03 11:23:35 +00:00
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: List of Pairs
"""
2021-12-03 11:12:33 +00:00
candle = ""
2021-12-08 13:35:15 +00:00
if candle_type != CandleType.SPOT:
datadir = datadir.joinpath('futures')
2021-12-03 11:12:33 +00:00
candle = f"-{candle_type}"
2021-12-03 11:12:33 +00:00
_tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.h5)', p.name)
for p in datadir.glob(f"*{timeframe}{candle}.h5")]
# Check if regex found something and only return these results
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
def ohlcv_store(
2021-12-07 19:30:58 +00:00
self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType) -> None:
"""
Store data in hdf5 file.
:param pair: Pair - used to generate filename
2021-06-25 17:13:31 +00:00
:param timeframe: Timeframe - used to generate filename
:param data: Dataframe containing OHLCV data
2021-12-03 11:23:35 +00:00
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: None
"""
key = self._pair_ohlcv_key(pair, timeframe)
_data = data.copy()
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
self.create_dir_if_needed(filename)
2020-07-25 15:06:58 +00:00
2021-12-01 19:32:23 +00:00
_data.loc[:, self._columns].to_hdf(
filename, key, mode='a', complevel=9, complib='blosc',
format='table', data_columns=['date']
)
def _ohlcv_load(self, pair: str, timeframe: str,
2021-12-07 19:30:58 +00:00
timerange: Optional[TimeRange], candle_type: CandleType
2021-12-03 11:23:35 +00:00
) -> pd.DataFrame:
"""
Internal method used to load data for one pair from disk.
Implements the loading and conversion to a Pandas dataframe.
Timerange trimming and dataframe validation happens outside of this method.
:param pair: Pair to load data
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
2021-12-03 11:23:35 +00:00
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame
"""
key = self._pair_ohlcv_key(pair, timeframe)
filename = self._pair_data_filename(
self._datadir,
pair,
timeframe,
candle_type=candle_type
)
if not filename.exists():
return pd.DataFrame(columns=self._columns)
where = []
if timerange:
if timerange.starttype == 'date':
where.append(f"date >= Timestamp({timerange.startts * 1e9})")
if timerange.stoptype == 'date':
where.append(f"date <= Timestamp({timerange.stopts * 1e9})")
pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
if list(pairdata.columns) != self._columns:
raise ValueError("Wrong dataframe format")
pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float',
'low': 'float', 'close': 'float', 'volume': 'float'})
return pairdata
def ohlcv_append(
self,
pair: str,
timeframe: str,
data: pd.DataFrame,
2021-12-03 11:23:35 +00:00
candle_type: CandleType
) -> None:
"""
Append data to existing data structures
:param pair: Pair
:param timeframe: Timeframe this ohlcv data is for
:param data: Data to append.
2021-12-03 11:23:35 +00:00
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
raise NotImplementedError()
@classmethod
def trades_get_pairs(cls, datadir: Path) -> List[str]:
"""
Returns a list of all pairs for which trade data is available in this
:param datadir: Directory to search for ohlcv files
:return: List of Pairs
"""
_tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name)
2020-07-12 18:41:25 +00:00
for p in datadir.glob("*trades.h5")]
# Check if regex found something and only return these results to avoid exceptions.
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
def trades_store(self, pair: str, data: TradeList) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Lists containing trade data,
column sequence as in DEFAULT_TRADES_COLUMNS
"""
key = self._pair_trades_key(pair)
2020-07-25 15:06:58 +00:00
2021-12-01 19:32:23 +00:00
pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf(
self._pair_trades_filename(self._datadir, pair), key,
mode='a', complevel=9, complib='blosc',
format='table', data_columns=['timestamp']
)
def trades_append(self, pair: str, data: TradeList):
"""
Append data to existing files
:param pair: Pair - used for filename
:param data: List of Lists containing trade data,
column sequence as in DEFAULT_TRADES_COLUMNS
"""
raise NotImplementedError()
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
"""
2020-07-12 18:41:25 +00:00
Load a pair from h5 file.
:param pair: Load trades for this pair
:param timerange: Timerange to load trades for - currently not implemented
:return: List of trades
"""
2020-07-12 18:41:25 +00:00
key = self._pair_trades_key(pair)
filename = self._pair_trades_filename(self._datadir, pair)
if not filename.exists():
return []
where = []
if timerange:
if timerange.starttype == 'date':
where.append(f"timestamp >= {timerange.startts * 1e3}")
if timerange.stoptype == 'date':
where.append(f"timestamp < {timerange.stopts * 1e3}")
trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where)
trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})
2020-07-12 18:41:25 +00:00
return trades.values.tolist()
@classmethod
def _get_file_extension(cls):
return "h5"
@classmethod
2020-07-25 15:19:41 +00:00
def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
# Escape futures pairs to avoid warnings
pair_esc = pair.replace(':', '_')
return f"{pair_esc}/ohlcv/tf_{timeframe}"
@classmethod
2020-07-25 15:19:41 +00:00
def _pair_trades_key(cls, pair: str) -> str:
return f"{pair}/trades"