2020-07-12 18:17:21 +00:00
|
|
|
import logging
|
|
|
|
import re
|
|
|
|
from pathlib import Path
|
|
|
|
from typing import List, Optional
|
|
|
|
|
2020-11-19 06:30:28 +00:00
|
|
|
import numpy as np
|
2020-07-12 18:17:21 +00:00
|
|
|
import pandas as pd
|
|
|
|
|
|
|
|
from freqtrade import misc
|
|
|
|
from freqtrade.configuration import TimeRange
|
2020-09-28 17:39:41 +00:00
|
|
|
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS,
|
2020-11-21 09:52:15 +00:00
|
|
|
ListPairsWithTimeframes, TradeList)
|
2020-07-12 18:17:21 +00:00
|
|
|
|
2020-11-21 09:52:15 +00:00
|
|
|
from .idatahandler import IDataHandler
|
2020-07-12 18:17:21 +00:00
|
|
|
|
2020-09-28 17:39:41 +00:00
|
|
|
|
2020-07-12 18:17:21 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
2020-07-24 17:23:37 +00:00
|
|
|
class HDF5DataHandler(IDataHandler):
|
2020-07-12 18:17:21 +00:00
|
|
|
|
|
|
|
_columns = DEFAULT_DATAFRAME_COLUMNS
|
|
|
|
|
2020-07-24 15:30:16 +00:00
|
|
|
@classmethod
|
|
|
|
def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
|
|
|
|
"""
|
|
|
|
Returns a list of all pairs with ohlcv data available in this datadir
|
|
|
|
:param datadir: Directory to search for ohlcv files
|
|
|
|
:return: List of Tuples of (pair, timeframe)
|
|
|
|
"""
|
2021-11-21 04:46:47 +00:00
|
|
|
_tmp = [
|
|
|
|
re.search(
|
2021-11-28 13:33:46 +00:00
|
|
|
cls._OHLCV_REGEX, p.name
|
2021-11-21 04:46:47 +00:00
|
|
|
) for p in datadir.glob("*.h5")
|
|
|
|
]
|
2021-11-28 14:03:55 +00:00
|
|
|
return [(cls.rebuild_pair_from_filename(match[1]), match[2], match[3]) for match in _tmp
|
2020-07-24 15:30:16 +00:00
|
|
|
if match and len(match.groups()) > 1]
|
|
|
|
|
2020-07-12 18:17:21 +00:00
|
|
|
@classmethod
|
2021-11-28 14:53:13 +00:00
|
|
|
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: str = '') -> List[str]:
|
2020-07-12 18:17:21 +00:00
|
|
|
"""
|
|
|
|
Returns a list of all pairs with ohlcv data available in this datadir
|
|
|
|
for the specified timeframe
|
|
|
|
:param datadir: Directory to search for ohlcv files
|
|
|
|
:param timeframe: Timeframe to search pairs for
|
2021-11-27 08:55:42 +00:00
|
|
|
:param candle_type: '', mark, index, premiumIndex, or funding_rate
|
2020-07-12 18:17:21 +00:00
|
|
|
:return: List of Pairs
|
|
|
|
"""
|
|
|
|
|
2021-11-07 06:35:27 +00:00
|
|
|
if candle_type:
|
|
|
|
candle_type = f"-{candle_type}"
|
|
|
|
else:
|
|
|
|
candle_type = ""
|
|
|
|
|
|
|
|
_tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle_type + '.h5)', p.name)
|
2021-11-19 10:55:50 +00:00
|
|
|
for p in datadir.glob(f"*{timeframe}{candle_type}.h5")]
|
2020-07-12 18:17:21 +00:00
|
|
|
# Check if regex found something and only return these results
|
|
|
|
return [match[0].replace('_', '/') for match in _tmp if match]
|
|
|
|
|
2021-11-07 06:35:27 +00:00
|
|
|
def ohlcv_store(
|
|
|
|
self,
|
|
|
|
pair: str,
|
|
|
|
timeframe: str,
|
|
|
|
data: pd.DataFrame,
|
2021-11-21 06:21:10 +00:00
|
|
|
candle_type: str = ''
|
2021-11-07 06:35:27 +00:00
|
|
|
) -> None:
|
2020-07-12 18:17:21 +00:00
|
|
|
"""
|
|
|
|
Store data in hdf5 file.
|
|
|
|
:param pair: Pair - used to generate filename
|
2021-06-25 17:13:31 +00:00
|
|
|
:param timeframe: Timeframe - used to generate filename
|
|
|
|
:param data: Dataframe containing OHLCV data
|
2021-11-27 08:55:42 +00:00
|
|
|
:param candle_type: '', mark, index, premiumIndex, or funding_rate
|
2020-07-12 18:17:21 +00:00
|
|
|
:return: None
|
|
|
|
"""
|
|
|
|
key = self._pair_ohlcv_key(pair, timeframe)
|
|
|
|
_data = data.copy()
|
|
|
|
|
2021-11-07 06:35:27 +00:00
|
|
|
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
|
2020-07-25 15:06:58 +00:00
|
|
|
|
2021-12-01 19:32:23 +00:00
|
|
|
_data.loc[:, self._columns].to_hdf(
|
|
|
|
filename, key, mode='a', complevel=9, complib='blosc',
|
|
|
|
format='table', data_columns=['date']
|
|
|
|
)
|
2020-07-12 18:17:21 +00:00
|
|
|
|
|
|
|
def _ohlcv_load(self, pair: str, timeframe: str,
|
2021-11-28 14:53:13 +00:00
|
|
|
timerange: Optional[TimeRange] = None, candle_type: str = '') -> pd.DataFrame:
|
2020-07-12 18:17:21 +00:00
|
|
|
"""
|
|
|
|
Internal method used to load data for one pair from disk.
|
|
|
|
Implements the loading and conversion to a Pandas dataframe.
|
|
|
|
Timerange trimming and dataframe validation happens outside of this method.
|
|
|
|
:param pair: Pair to load data
|
|
|
|
:param timeframe: Timeframe (e.g. "5m")
|
|
|
|
:param timerange: Limit data to be loaded to this timerange.
|
|
|
|
Optionally implemented by subclasses to avoid loading
|
|
|
|
all data where possible.
|
2021-11-27 08:55:42 +00:00
|
|
|
:param candle_type: '', mark, index, premiumIndex, or funding_rate
|
2020-07-12 18:17:21 +00:00
|
|
|
:return: DataFrame with ohlcv data, or empty DataFrame
|
|
|
|
"""
|
|
|
|
key = self._pair_ohlcv_key(pair, timeframe)
|
2021-11-07 06:35:27 +00:00
|
|
|
filename = self._pair_data_filename(
|
|
|
|
self._datadir,
|
|
|
|
pair,
|
|
|
|
timeframe,
|
|
|
|
candle_type=candle_type
|
|
|
|
)
|
2020-07-12 18:17:21 +00:00
|
|
|
|
|
|
|
if not filename.exists():
|
|
|
|
return pd.DataFrame(columns=self._columns)
|
|
|
|
where = []
|
|
|
|
if timerange:
|
|
|
|
if timerange.starttype == 'date':
|
|
|
|
where.append(f"date >= Timestamp({timerange.startts * 1e9})")
|
|
|
|
if timerange.stoptype == 'date':
|
2021-04-24 18:26:37 +00:00
|
|
|
where.append(f"date <= Timestamp({timerange.stopts * 1e9})")
|
2020-07-12 18:17:21 +00:00
|
|
|
|
|
|
|
pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
|
|
|
|
|
|
|
|
if list(pairdata.columns) != self._columns:
|
|
|
|
raise ValueError("Wrong dataframe format")
|
|
|
|
pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float',
|
|
|
|
'low': 'float', 'close': 'float', 'volume': 'float'})
|
|
|
|
return pairdata
|
|
|
|
|
2021-11-21 06:21:10 +00:00
|
|
|
def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool:
|
2020-07-12 18:17:21 +00:00
|
|
|
"""
|
|
|
|
Remove data for this pair
|
|
|
|
:param pair: Delete data for this pair.
|
|
|
|
:param timeframe: Timeframe (e.g. "5m")
|
2021-11-27 08:55:42 +00:00
|
|
|
:param candle_type: '', mark, index, premiumIndex, or funding_rate
|
2020-07-12 18:17:21 +00:00
|
|
|
:return: True when deleted, false if file did not exist.
|
|
|
|
"""
|
2021-11-07 06:35:27 +00:00
|
|
|
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
|
2020-07-12 18:17:21 +00:00
|
|
|
if filename.exists():
|
|
|
|
filename.unlink()
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
2021-11-07 06:35:27 +00:00
|
|
|
def ohlcv_append(
|
|
|
|
self,
|
|
|
|
pair: str,
|
|
|
|
timeframe: str,
|
|
|
|
data: pd.DataFrame,
|
2021-11-21 06:21:10 +00:00
|
|
|
candle_type: str = ''
|
2021-11-07 06:35:27 +00:00
|
|
|
) -> None:
|
2020-07-12 18:17:21 +00:00
|
|
|
"""
|
|
|
|
Append data to existing data structures
|
|
|
|
:param pair: Pair
|
|
|
|
:param timeframe: Timeframe this ohlcv data is for
|
|
|
|
:param data: Data to append.
|
2021-11-27 08:55:42 +00:00
|
|
|
:param candle_type: '', mark, index, premiumIndex, or funding_rate
|
2020-07-12 18:17:21 +00:00
|
|
|
"""
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def trades_get_pairs(cls, datadir: Path) -> List[str]:
|
|
|
|
"""
|
|
|
|
Returns a list of all pairs for which trade data is available in this
|
|
|
|
:param datadir: Directory to search for ohlcv files
|
|
|
|
:return: List of Pairs
|
|
|
|
"""
|
|
|
|
_tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name)
|
2020-07-12 18:41:25 +00:00
|
|
|
for p in datadir.glob("*trades.h5")]
|
2020-07-12 18:17:21 +00:00
|
|
|
# Check if regex found something and only return these results to avoid exceptions.
|
|
|
|
return [match[0].replace('_', '/') for match in _tmp if match]
|
|
|
|
|
|
|
|
def trades_store(self, pair: str, data: TradeList) -> None:
|
|
|
|
"""
|
|
|
|
Store trades data (list of Dicts) to file
|
|
|
|
:param pair: Pair - used for filename
|
|
|
|
:param data: List of Lists containing trade data,
|
|
|
|
column sequence as in DEFAULT_TRADES_COLUMNS
|
|
|
|
"""
|
|
|
|
key = self._pair_trades_key(pair)
|
2020-07-25 15:06:58 +00:00
|
|
|
|
2021-12-01 19:32:23 +00:00
|
|
|
pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf(
|
|
|
|
self._pair_trades_filename(self._datadir, pair), key,
|
|
|
|
mode='a', complevel=9, complib='blosc',
|
|
|
|
format='table', data_columns=['timestamp']
|
|
|
|
)
|
2020-07-12 18:17:21 +00:00
|
|
|
|
|
|
|
def trades_append(self, pair: str, data: TradeList):
|
|
|
|
"""
|
|
|
|
Append data to existing files
|
|
|
|
:param pair: Pair - used for filename
|
|
|
|
:param data: List of Lists containing trade data,
|
|
|
|
column sequence as in DEFAULT_TRADES_COLUMNS
|
|
|
|
"""
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
|
|
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
|
|
|
|
"""
|
2020-07-12 18:41:25 +00:00
|
|
|
Load a pair from h5 file.
|
2020-07-12 18:17:21 +00:00
|
|
|
:param pair: Load trades for this pair
|
|
|
|
:param timerange: Timerange to load trades for - currently not implemented
|
|
|
|
:return: List of trades
|
|
|
|
"""
|
2020-07-12 18:41:25 +00:00
|
|
|
key = self._pair_trades_key(pair)
|
|
|
|
filename = self._pair_trades_filename(self._datadir, pair)
|
|
|
|
|
|
|
|
if not filename.exists():
|
|
|
|
return []
|
|
|
|
where = []
|
|
|
|
if timerange:
|
|
|
|
if timerange.starttype == 'date':
|
|
|
|
where.append(f"timestamp >= {timerange.startts * 1e3}")
|
|
|
|
if timerange.stoptype == 'date':
|
|
|
|
where.append(f"timestamp < {timerange.stopts * 1e3}")
|
|
|
|
|
2020-11-19 06:30:28 +00:00
|
|
|
trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where)
|
|
|
|
trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})
|
2020-07-12 18:41:25 +00:00
|
|
|
return trades.values.tolist()
|
2020-07-12 18:17:21 +00:00
|
|
|
|
|
|
|
def trades_purge(self, pair: str) -> bool:
|
|
|
|
"""
|
|
|
|
Remove data for this pair
|
|
|
|
:param pair: Delete data for this pair.
|
|
|
|
:return: True when deleted, false if file did not exist.
|
|
|
|
"""
|
|
|
|
filename = self._pair_trades_filename(self._datadir, pair)
|
|
|
|
if filename.exists():
|
|
|
|
filename.unlink()
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
|
|
@classmethod
|
2020-07-25 15:19:41 +00:00
|
|
|
def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
|
2020-07-12 18:17:21 +00:00
|
|
|
return f"{pair}/ohlcv/tf_{timeframe}"
|
|
|
|
|
|
|
|
@classmethod
|
2020-07-25 15:19:41 +00:00
|
|
|
def _pair_trades_key(cls, pair: str) -> str:
|
2020-07-12 18:17:21 +00:00
|
|
|
return f"{pair}/trades"
|
|
|
|
|
|
|
|
@classmethod
|
2021-11-07 06:35:27 +00:00
|
|
|
def _pair_data_filename(
|
|
|
|
cls,
|
|
|
|
datadir: Path,
|
|
|
|
pair: str,
|
|
|
|
timeframe: str,
|
2021-11-21 06:21:10 +00:00
|
|
|
candle_type: str = ''
|
2021-11-07 06:35:27 +00:00
|
|
|
) -> Path:
|
2020-07-12 18:17:21 +00:00
|
|
|
pair_s = misc.pair_to_filename(pair)
|
2021-11-07 06:35:27 +00:00
|
|
|
if candle_type:
|
|
|
|
candle_type = f"-{candle_type}"
|
|
|
|
filename = datadir.joinpath(f'{pair_s}-{timeframe}{candle_type}.h5')
|
2020-07-12 18:17:21 +00:00
|
|
|
return filename
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
|
|
|
|
pair_s = misc.pair_to_filename(pair)
|
|
|
|
filename = datadir.joinpath(f'{pair_s}-trades.h5')
|
|
|
|
return filename
|