diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 396b55ef5..44bc71038 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -65,9 +65,9 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_c ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", "hyperopt_show_no_header"] -NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs", - "list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe", - "plot-profit"] +NO_CONF_REQURIED = ["convert-data", "download-data", "list-timeframes", "list-markets", + "list-pairs", "list-strategies", "hyperopt-list", "hyperopt-show", + "plot-dataframe", "plot-profit"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] diff --git a/freqtrade/data/datahandlers/__init__.py b/freqtrade/data/datahandlers/__init__.py new file mode 100644 index 000000000..b3aa2e3a4 --- /dev/null +++ b/freqtrade/data/datahandlers/__init__.py @@ -0,0 +1,20 @@ +from .idatahandler import IDataHandler + + +def get_datahandlerclass(datatype: str) -> IDataHandler: + """ + Get datahandler class. + Could be done using Resolvers, but since this may be called often and resolvers + are rather expensive, doing this directly should improve performance. + :param datatype: datatype to use. + :return: Datahandler class + """ + + if datatype == 'json': + from .jsondatahandler import JsonDataHandler + return JsonDataHandler + elif datatype == 'jsongz': + from .jsondatahandler import JsonGzDataHandler + return JsonGzDataHandler + else: + raise ValueError(f"No datahandler for datatype {datatype} available.") diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py new file mode 100644 index 000000000..ffe50b14e --- /dev/null +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -0,0 +1,97 @@ +""" +Abstract datahandler interface. +It's subclasses handle and storing data from disk. + +""" + +from abc import ABC, abstractmethod, abstractclassmethod +from pathlib import Path +from typing import Dict, List, Optional + +from pandas import DataFrame + +from freqtrade.configuration import TimeRange + + +class IDataHandler(ABC): + + def __init__(self, datadir: Path, pair: str) -> None: + self._datadir = datadir + self._pair = pair + + @abstractclassmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + + @abstractmethod + def ohlcv_store(self, timeframe: str, data: DataFrame): + """ + Store data + """ + + @abstractmethod + def ohlcv_append(self, timeframe: str, data: DataFrame): + """ + Append data to existing files + """ + + @abstractmethod + def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame: + """ + Load data for one pair + :return: Dataframe + """ + + @abstractclassmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + + @abstractmethod + def trades_store(self, data: DataFrame): + """ + Store data + """ + + @abstractmethod + def trades_append(self, data: DataFrame): + """ + Append data to existing files + """ + + @abstractmethod + def trades_load(self, timerange: Optional[TimeRange] = None): + """ + Load data for one pair + :return: Dataframe + """ + + @staticmethod + def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: + """ + TODO: investigate if this is needed ... we can probably cover this in a dataframe + Trim tickerlist based on given timerange + """ + if not tickerlist: + return tickerlist + + start_index = 0 + stop_index = len(tickerlist) + + if timerange.starttype == 'date': + while (start_index < len(tickerlist) and + tickerlist[start_index][0] < timerange.startts * 1000): + start_index += 1 + + if timerange.stoptype == 'date': + while (stop_index > 0 and + tickerlist[stop_index-1][0] > timerange.stopts * 1000): + stop_index -= 1 + + if start_index > stop_index: + raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') + + return tickerlist[start_index:stop_index] diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py new file mode 100644 index 000000000..214958251 --- /dev/null +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -0,0 +1,105 @@ +import re +from pathlib import Path +from typing import Dict, List, Optional + +from pandas import DataFrame + +from freqtrade import misc +from freqtrade.configuration import TimeRange + +from .idatahandler import IDataHandler + + +class JsonDataHandler(IDataHandler): + + _use_zip = False + + @classmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + return [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name)[0].replace('_', ' /') + for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] + + def ohlcv_store(self, timeframe: str, data: DataFrame): + """ + Store data + """ + raise NotImplementedError() + + def ohlcv_append(self, timeframe: str, data: DataFrame): + """ + Append data to existing files + """ + raise NotImplementedError() + + def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame: + """ + Load data for one pair + :return: Dataframe + """ + filename = JsonDataHandler._pair_data_filename(self.datadir, self._pair, + self._pair, timeframe) + pairdata = misc.file_load_json(filename) + if not pairdata: + return [] + + if timerange: + pairdata = IDataHandler.trim_tickerlist(pairdata, timerange) + return pairdata + + @classmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + return [re.search(r'^(\S+)(?=\-trades.json)', p.name)[0].replace('_', '/') + for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] + + def trades_store(self, data: List[Dict]): + """ + Store data + """ + filename = self._pair_trades_filename(self._datadir, self._pair) + misc.file_dump_json(filename, data, is_zip=self._use_zip) + + def trades_append(self, data: DataFrame): + """ + Append data to existing files + """ + raise NotImplementedError() + + def trades_load(self, timerange: Optional[TimeRange] = None) -> List[Dict]: + """ + Load a pair from file, either .json.gz or .json + # TODO: validate timerange ... + :return: List of trades + """ + filename = self._pair_trades_filename(self._datadir, self._pair) + tradesdata = misc.file_load_json(filename) + if not tradesdata: + return [] + + return tradesdata + + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = pair.replace("/", "_") + filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') + return filename + + @classmethod + def _get_file_extension(cls): + return "json.gz" if cls._use_zip else "json" + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = pair.replace("/", "_") + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') + return filename + + +class JsonGzDataHandler(JsonDataHandler): + + _use_zip = True