Merge pull request #3582 from freqtrade/data/list

List available backtesting data

freqtrade/commands/__init__.py

@@ -9,7 +9,8 @@ Note: Be careful with file-scoped imports in these subfiles.
 from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
 from freqtrade.commands.data_commands import (start_convert_data,
-                                              start_download_data)
+                                              start_download_data,
+                                              start_list_data)
 from freqtrade.commands.deploy_commands import (start_create_userdir,
                                                 start_new_hyperopt,
                                                 start_new_strategy)

freqtrade/commands/arguments.py

@@ -54,6 +54,8 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
+ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
+
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
                       "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
 
@@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
                       "print_json", "hyperopt_show_no_header"]
 
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
-                    "list-markets", "list-pairs", "list-strategies",
+                    "list-markets", "list-pairs", "list-strategies", "list-data",
                     "list-hyperopts", "hyperopt-list", "hyperopt-show",
                     "plot-dataframe", "plot-profit", "show-trades"]
 
@@ -159,7 +161,7 @@ class Arguments:
         self._build_args(optionlist=['version'], parser=self.parser)
 
         from freqtrade.commands import (start_create_userdir, start_convert_data,
-                                        start_download_data,
+                                        start_download_data, start_list_data,
                                         start_hyperopt_list, start_hyperopt_show,
                                         start_list_exchanges, start_list_hyperopts,
                                         start_list_markets, start_list_strategies,
@@ -233,6 +235,15 @@ class Arguments:
         convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
         self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
 
+        # Add list-data subcommand
+        list_data_cmd = subparsers.add_parser(
+            'list-data',
+            help='List downloaded data.',
+            parents=[_common_parser],
+        )
+        list_data_cmd.set_defaults(func=start_list_data)
+        self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
+
         # Add backtesting subcommand
         backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
                                                 parents=[_common_parser, _strategy_parser])

freqtrade/commands/data_commands.py

@@ -1,5 +1,6 @@
 import logging
 import sys
+from collections import defaultdict
 from typing import Any, Dict, List
 
 import arrow
@@ -11,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv,
                                     refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
 from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
 
@@ -88,3 +90,30 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
         convert_trades_format(config,
                               convert_from=args['format_from'], convert_to=args['format_to'],
                               erase=args['erase'])
+
+
+def start_list_data(args: Dict[str, Any]) -> None:
+    """
+    List available backtest data
+    """
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    from freqtrade.data.history.idatahandler import get_datahandler
+    from tabulate import tabulate
+    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
+
+    paircombs = dhc.ohlcv_get_available_data(config['datadir'])
+
+    if args['pairs']:
+        paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
+
+    print(f"Found {len(paircombs)} pair / timeframe combinations.")
+    groupedpair = defaultdict(list)
+    for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))):
+        groupedpair[pair].append(timeframe)
+
+    if groupedpair:
+        print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()],
+                       headers=("Pair", "Timeframe"),
+                       tablefmt='psql', stralign='right'))

freqtrade/data/history/idatahandler.py

@@ -13,6 +13,7 @@ from typing import List, Optional, Type
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
+from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.data.converter import (clean_ohlcv_dataframe,
                                       trades_remove_duplicates, trim_dataframe)
 from freqtrade.exchange import timeframe_to_seconds
@@ -28,6 +29,14 @@ class IDataHandler(ABC):
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
 
+    @abstractclassmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+
     @abstractclassmethod
     def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
         """

freqtrade/data/history/jsondatahandler.py

@@ -8,7 +8,8 @@ from pandas import DataFrame, read_json, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
+                                 ListPairsWithTimeframes)
 from freqtrade.data.converter import trades_dict_to_list
 
 from .idatahandler import IDataHandler, TradeList
@@ -21,6 +22,18 @@ class JsonDataHandler(IDataHandler):
     _use_zip = False
     _columns = DEFAULT_DATAFRAME_COLUMNS
 
+    @classmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+        _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name)
+                for p in datadir.glob(f"*.{cls._get_file_extension()}")]
+        return [(match[1].replace('_', '/'), match[2]) for match in _tmp
+                if match and len(match.groups()) > 1]
+
     @classmethod
     def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
         """