Implement first version of jsondatahandler

This commit is contained in:
Matthias 2019-12-23 14:56:48 +01:00
parent 2496aa8e3f
commit e5a61667dd
4 changed files with 225 additions and 3 deletions

View File

@ -65,9 +65,9 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_c
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
"print_json", "hyperopt_show_no_header"] "print_json", "hyperopt_show_no_header"]
NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs", NO_CONF_REQURIED = ["convert-data", "download-data", "list-timeframes", "list-markets",
"list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe", "list-pairs", "list-strategies", "hyperopt-list", "hyperopt-show",
"plot-profit"] "plot-dataframe", "plot-profit"]
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]

View File

@ -0,0 +1,20 @@
from .idatahandler import IDataHandler
def get_datahandlerclass(datatype: str) -> IDataHandler:
"""
Get datahandler class.
Could be done using Resolvers, but since this may be called often and resolvers
are rather expensive, doing this directly should improve performance.
:param datatype: datatype to use.
:return: Datahandler class
"""
if datatype == 'json':
from .jsondatahandler import JsonDataHandler
return JsonDataHandler
elif datatype == 'jsongz':
from .jsondatahandler import JsonGzDataHandler
return JsonGzDataHandler
else:
raise ValueError(f"No datahandler for datatype {datatype} available.")

View File

@ -0,0 +1,97 @@
"""
Abstract datahandler interface.
It's subclasses handle and storing data from disk.
"""
from abc import ABC, abstractmethod, abstractclassmethod
from pathlib import Path
from typing import Dict, List, Optional
from pandas import DataFrame
from freqtrade.configuration import TimeRange
class IDataHandler(ABC):
def __init__(self, datadir: Path, pair: str) -> None:
self._datadir = datadir
self._pair = pair
@abstractclassmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
"""
Returns a list of all pairs available in this datadir
"""
@abstractmethod
def ohlcv_store(self, timeframe: str, data: DataFrame):
"""
Store data
"""
@abstractmethod
def ohlcv_append(self, timeframe: str, data: DataFrame):
"""
Append data to existing files
"""
@abstractmethod
def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame:
"""
Load data for one pair
:return: Dataframe
"""
@abstractclassmethod
def trades_get_pairs(cls, datadir: Path) -> List[str]:
"""
Returns a list of all pairs available in this datadir
"""
@abstractmethod
def trades_store(self, data: DataFrame):
"""
Store data
"""
@abstractmethod
def trades_append(self, data: DataFrame):
"""
Append data to existing files
"""
@abstractmethod
def trades_load(self, timerange: Optional[TimeRange] = None):
"""
Load data for one pair
:return: Dataframe
"""
@staticmethod
def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
"""
TODO: investigate if this is needed ... we can probably cover this in a dataframe
Trim tickerlist based on given timerange
"""
if not tickerlist:
return tickerlist
start_index = 0
stop_index = len(tickerlist)
if timerange.starttype == 'date':
while (start_index < len(tickerlist) and
tickerlist[start_index][0] < timerange.startts * 1000):
start_index += 1
if timerange.stoptype == 'date':
while (stop_index > 0 and
tickerlist[stop_index-1][0] > timerange.stopts * 1000):
stop_index -= 1
if start_index > stop_index:
raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect')
return tickerlist[start_index:stop_index]

View File

@ -0,0 +1,105 @@
import re
from pathlib import Path
from typing import Dict, List, Optional
from pandas import DataFrame
from freqtrade import misc
from freqtrade.configuration import TimeRange
from .idatahandler import IDataHandler
class JsonDataHandler(IDataHandler):
_use_zip = False
@classmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
"""
Returns a list of all pairs available in this datadir
"""
return [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name)[0].replace('_', ' /')
for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")]
def ohlcv_store(self, timeframe: str, data: DataFrame):
"""
Store data
"""
raise NotImplementedError()
def ohlcv_append(self, timeframe: str, data: DataFrame):
"""
Append data to existing files
"""
raise NotImplementedError()
def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame:
"""
Load data for one pair
:return: Dataframe
"""
filename = JsonDataHandler._pair_data_filename(self.datadir, self._pair,
self._pair, timeframe)
pairdata = misc.file_load_json(filename)
if not pairdata:
return []
if timerange:
pairdata = IDataHandler.trim_tickerlist(pairdata, timerange)
return pairdata
@classmethod
def trades_get_pairs(cls, datadir: Path) -> List[str]:
"""
Returns a list of all pairs available in this datadir
"""
return [re.search(r'^(\S+)(?=\-trades.json)', p.name)[0].replace('_', '/')
for p in datadir.glob(f"*trades.{cls._get_file_extension()}")]
def trades_store(self, data: List[Dict]):
"""
Store data
"""
filename = self._pair_trades_filename(self._datadir, self._pair)
misc.file_dump_json(filename, data, is_zip=self._use_zip)
def trades_append(self, data: DataFrame):
"""
Append data to existing files
"""
raise NotImplementedError()
def trades_load(self, timerange: Optional[TimeRange] = None) -> List[Dict]:
"""
Load a pair from file, either .json.gz or .json
# TODO: validate timerange ...
:return: List of trades
"""
filename = self._pair_trades_filename(self._datadir, self._pair)
tradesdata = misc.file_load_json(filename)
if not tradesdata:
return []
return tradesdata
@classmethod
def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path:
pair_s = pair.replace("/", "_")
filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}')
return filename
@classmethod
def _get_file_extension(cls):
return "json.gz" if cls._use_zip else "json"
@classmethod
def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
pair_s = pair.replace("/", "_")
filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
return filename
class JsonGzDataHandler(JsonDataHandler):
_use_zip = True