Implement first version of jsondatahandler

freqtrade/configuration/arguments.py

@@ -65,9 +65,9 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_c
 ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
                       "print_json", "hyperopt_show_no_header"]
 
-NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs",
+NO_CONF_REQURIED = ["convert-data", "download-data", "list-timeframes", "list-markets",
-                    "list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe",
+                    "list-pairs", "list-strategies", "hyperopt-list", "hyperopt-show",
-                    "plot-profit"]
+                    "plot-dataframe", "plot-profit"]
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]
 

freqtrade/data/datahandlers/__init__.py

@@ -0,0 +1,20 @@
+from .idatahandler import IDataHandler
+
+
+def get_datahandlerclass(datatype: str) -> IDataHandler:
+    """
+    Get datahandler class.
+    Could be done using Resolvers, but since this may be called often and resolvers
+    are rather expensive, doing this directly should improve performance.
+    :param datatype: datatype to use.
+    :return: Datahandler class
+    """
+
+    if datatype == 'json':
+        from .jsondatahandler import JsonDataHandler
+        return JsonDataHandler
+    elif datatype == 'jsongz':
+        from .jsondatahandler import JsonGzDataHandler
+        return JsonGzDataHandler
+    else:
+        raise ValueError(f"No datahandler for datatype {datatype} available.")

freqtrade/data/datahandlers/idatahandler.py

@@ -0,0 +1,97 @@
+"""
+Abstract datahandler interface.
+It's subclasses handle and storing data from disk.
+
+"""
+
+from abc import ABC, abstractmethod, abstractclassmethod
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from pandas import DataFrame
+
+from freqtrade.configuration import TimeRange
+
+
+class IDataHandler(ABC):
+
+    def __init__(self, datadir: Path, pair: str) -> None:
+        self._datadir = datadir
+        self._pair = pair
+
+    @abstractclassmethod
+    def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
+        """
+        Returns a list of all pairs available in this datadir
+        """
+
+    @abstractmethod
+    def ohlcv_store(self, timeframe: str, data: DataFrame):
+        """
+        Store data
+        """
+
+    @abstractmethod
+    def ohlcv_append(self, timeframe: str, data: DataFrame):
+        """
+        Append data to existing files
+        """
+
+    @abstractmethod
+    def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+        """
+        Load data for one pair
+        :return: Dataframe
+        """
+
+    @abstractclassmethod
+    def trades_get_pairs(cls, datadir: Path) -> List[str]:
+        """
+        Returns a list of all pairs available in this datadir
+        """
+
+    @abstractmethod
+    def trades_store(self, data: DataFrame):
+        """
+        Store data
+        """
+
+    @abstractmethod
+    def trades_append(self, data: DataFrame):
+        """
+        Append data to existing files
+        """
+
+    @abstractmethod
+    def trades_load(self, timerange: Optional[TimeRange] = None):
+        """
+        Load data for one pair
+        :return: Dataframe
+        """
+
+    @staticmethod
+    def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
+        """
+        TODO: investigate if this is needed ... we can probably cover this in a dataframe
+        Trim tickerlist based on given timerange
+        """
+        if not tickerlist:
+            return tickerlist
+
+        start_index = 0
+        stop_index = len(tickerlist)
+
+        if timerange.starttype == 'date':
+            while (start_index < len(tickerlist) and
+                   tickerlist[start_index][0] < timerange.startts * 1000):
+                start_index += 1
+
+        if timerange.stoptype == 'date':
+            while (stop_index > 0 and
+                   tickerlist[stop_index-1][0] > timerange.stopts * 1000):
+                stop_index -= 1
+
+        if start_index > stop_index:
+            raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect')
+
+        return tickerlist[start_index:stop_index]

freqtrade/data/datahandlers/jsondatahandler.py

@@ -0,0 +1,105 @@
+import re
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from pandas import DataFrame
+
+from freqtrade import misc
+from freqtrade.configuration import TimeRange
+
+from .idatahandler import IDataHandler
+
+
+class JsonDataHandler(IDataHandler):
+
+    _use_zip = False
+
+    @classmethod
+    def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
+        """
+        Returns a list of all pairs available in this datadir
+        """
+        return [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name)[0].replace('_', ' /')
+                for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")]
+
+    def ohlcv_store(self, timeframe: str, data: DataFrame):
+        """
+        Store data
+        """
+        raise NotImplementedError()
+
+    def ohlcv_append(self, timeframe: str, data: DataFrame):
+        """
+        Append data to existing files
+        """
+        raise NotImplementedError()
+
+    def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+        """
+        Load data for one pair
+        :return: Dataframe
+        """
+        filename = JsonDataHandler._pair_data_filename(self.datadir, self._pair,
+                                                       self._pair, timeframe)
+        pairdata = misc.file_load_json(filename)
+        if not pairdata:
+            return []
+
+        if timerange:
+            pairdata = IDataHandler.trim_tickerlist(pairdata, timerange)
+        return pairdata
+
+    @classmethod
+    def trades_get_pairs(cls, datadir: Path) -> List[str]:
+        """
+        Returns a list of all pairs available in this datadir
+        """
+        return [re.search(r'^(\S+)(?=\-trades.json)', p.name)[0].replace('_', '/')
+                for p in datadir.glob(f"*trades.{cls._get_file_extension()}")]
+
+    def trades_store(self, data: List[Dict]):
+        """
+        Store data
+        """
+        filename = self._pair_trades_filename(self._datadir, self._pair)
+        misc.file_dump_json(filename, data, is_zip=self._use_zip)
+
+    def trades_append(self, data: DataFrame):
+        """
+        Append data to existing files
+        """
+        raise NotImplementedError()
+
+    def trades_load(self, timerange: Optional[TimeRange] = None) -> List[Dict]:
+        """
+        Load a pair from file, either .json.gz or .json
+        # TODO: validate timerange ...
+        :return: List of trades
+        """
+        filename = self._pair_trades_filename(self._datadir, self._pair)
+        tradesdata = misc.file_load_json(filename)
+        if not tradesdata:
+            return []
+
+        return tradesdata
+
+    @classmethod
+    def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path:
+        pair_s = pair.replace("/", "_")
+        filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}')
+        return filename
+
+    @classmethod
+    def _get_file_extension(cls):
+        return "json.gz" if cls._use_zip else "json"
+
+    @classmethod
+    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
+        pair_s = pair.replace("/", "_")
+        filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
+        return filename
+
+
+class JsonGzDataHandler(JsonDataHandler):
+
+    _use_zip = True