adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Load and save using pandas internal function

Browse Source
This commit is contained in:
Matthias 2019-12-25 15:05:01 +01:00
parent 377d59abe7
commit 866908d2ca
1 changed files with 42 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
freqtrade/data/datahandlers/jsondatahandler.py
View File

@ -2,11 +2,12 @@ import re
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional from typing import Dict, List, Optional
from pandas import DataFrame import numpy as np
from pandas import DataFrame, read_json, to_datetime
from freqtrade import misc from freqtrade import misc
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.converter import clean_ohlcv_dataframe
from .idatahandler import IDataHandler from .idatahandler import IDataHandler
@ -14,6 +15,7 @@ from .idatahandler import IDataHandler
class JsonDataHandler(IDataHandler): class JsonDataHandler(IDataHandler):
_use_zip = False _use_zip = False
_columns = ['date', 'open', 'high', 'low', 'close', 'volume']
@classmethod @classmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
@ -28,20 +30,27 @@ class JsonDataHandler(IDataHandler):
def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None:
""" """
Store data Store data in json format "values".
format looks as follows:
[[<date>,<open>,<high>,<low>,<close>]]
:param pair: Pair - used to generate filename
:timeframe: Timeframe - used to generate filename
:data: Dataframe containing OHLCV data
:return: None
""" """
filename = JsonDataHandler._pair_data_filename(self._datadir, pair, timeframe) filename = self._pair_data_filename(self._datadir, pair, timeframe)
misc.file_dump_json(filename, data, is_zip=self._use_zip) _data = data.copy()
# Convert date to int
_data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: # Reset index, select only appropriate columns and save as json
""" _data.reset_index(drop=True).loc[:, self._columns].to_json(
Append data to existing files filename, orient="values",
""" compression='gzip' if self._use_zip else None)
raise NotImplementedError()
def _ohlcv_load(self, pair: str, timeframe: str, def _ohlcv_load(self, pair: str, timeframe: str,
timerange: Optional[TimeRange] = None, timerange: Optional[TimeRange] = None,
fill_up_missing: bool = True, fill_missing: bool = True,
drop_incomplete: bool = True, drop_incomplete: bool = True,
) -> DataFrame: ) -> DataFrame:
""" """
@ -49,18 +58,31 @@ class JsonDataHandler(IDataHandler):
Implements the loading and conversation to a Pandas dataframe. Implements the loading and conversation to a Pandas dataframe.
:return: Dataframe :return: Dataframe
""" """
filename = JsonDataHandler._pair_data_filename(self._datadir, pair, timeframe) filename = self._pair_data_filename(self._datadir, pair, timeframe)
pairdata = misc.file_load_json(filename) pairdata = read_json(filename, orient='values')
if not pairdata: pairdata.columns = self._columns
return DataFrame() pairdata['date'] = to_datetime(pairdata['date'],
unit='ms',
utc=True,
infer_datetime_format=True)
if timerange: if timerange:
pairdata = IDataHandler.trim_tickerlist(pairdata, timerange) pairdata = IDataHandler.trim_tickerlist(pairdata, timerange)
return parse_ticker_dataframe(pairdata, timeframe,
pair=self._pair, return clean_ohlcv_dataframe(pairdata, timeframe,
fill_missing=fill_up_missing, pair=pair,
fill_missing=fill_missing,
drop_incomplete=drop_incomplete) drop_incomplete=drop_incomplete)
return pairdata
def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None:
"""
Append data to existing data structures
:param pair: Pair
:param timeframe: Timeframe this ohlcv data is for
:param data: Data to append.
"""
raise NotImplementedError()
@classmethod @classmethod
def trades_get_pairs(cls, datadir: Path) -> List[str]: def trades_get_pairs(cls, datadir: Path) -> List[str]: