Merge pull request #3582 from freqtrade/data/list

List available backtesting data

docs/data-download.md

@@ -158,6 +158,58 @@ It'll also remove original jsongz data files (`--erase` parameter).
 freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
 ```
 
+### Subcommand list-data
+
+You can get a list of downloaded data using the `list-data` subcommand.
+
+```
+usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
+                           [--userdir PATH] [--exchange EXCHANGE]
+                           [--data-format-ohlcv {json,jsongz}]
+                           [-p PAIRS [PAIRS ...]]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
+  --data-format-ohlcv {json,jsongz}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `json`).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Show profits for only these pairs. Pairs are space-
+                        separated.
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+```
+
+#### Example list-data
+
+```bash
+> freqtrade list-data --userdir ~/.freqtrade/user_data/
+
+Found 33 pair / timeframe combinations.
+pairs       timeframe
+----------  -----------------------------------------
+ADA/BTC     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+ADA/ETH     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+ETH/BTC     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+ETH/USDT    5m, 15m, 30m, 1h, 2h, 4h
+```
+
 ### Pairs file
 
 In alternative to the whitelist from `config.json`, a `pairs.json` file can be used.

freqtrade/commands/__init__.py

@@ -9,7 +9,8 @@ Note: Be careful with file-scoped imports in these subfiles.
 from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
 from freqtrade.commands.data_commands import (start_convert_data,
-                                              start_download_data)
+                                              start_download_data,
+                                              start_list_data)
 from freqtrade.commands.deploy_commands import (start_create_userdir,
                                                 start_new_hyperopt,
                                                 start_new_strategy)

freqtrade/commands/arguments.py

@@ -54,6 +54,8 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
+ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
+
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
                       "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
 
@@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
                       "print_json", "hyperopt_show_no_header"]
 
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
-                    "list-markets", "list-pairs", "list-strategies",
+                    "list-markets", "list-pairs", "list-strategies", "list-data",
                     "list-hyperopts", "hyperopt-list", "hyperopt-show",
                     "plot-dataframe", "plot-profit", "show-trades"]
 
@@ -159,7 +161,7 @@ class Arguments:
         self._build_args(optionlist=['version'], parser=self.parser)
 
         from freqtrade.commands import (start_create_userdir, start_convert_data,
-                                        start_download_data,
+                                        start_download_data, start_list_data,
                                         start_hyperopt_list, start_hyperopt_show,
                                         start_list_exchanges, start_list_hyperopts,
                                         start_list_markets, start_list_strategies,
@@ -233,6 +235,15 @@ class Arguments:
         convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
         self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
 
+        # Add list-data subcommand
+        list_data_cmd = subparsers.add_parser(
+            'list-data',
+            help='List downloaded data.',
+            parents=[_common_parser],
+        )
+        list_data_cmd.set_defaults(func=start_list_data)
+        self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
+
         # Add backtesting subcommand
         backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
                                                 parents=[_common_parser, _strategy_parser])

freqtrade/commands/data_commands.py

@@ -1,5 +1,6 @@
 import logging
 import sys
+from collections import defaultdict
 from typing import Any, Dict, List
 
 import arrow
@@ -11,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv,
                                     refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
 from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
 
@@ -88,3 +90,30 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
         convert_trades_format(config,
                               convert_from=args['format_from'], convert_to=args['format_to'],
                               erase=args['erase'])
+
+
+def start_list_data(args: Dict[str, Any]) -> None:
+    """
+    List available backtest data
+    """
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    from freqtrade.data.history.idatahandler import get_datahandler
+    from tabulate import tabulate
+    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
+
+    paircombs = dhc.ohlcv_get_available_data(config['datadir'])
+
+    if args['pairs']:
+        paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
+
+    print(f"Found {len(paircombs)} pair / timeframe combinations.")
+    groupedpair = defaultdict(list)
+    for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))):
+        groupedpair[pair].append(timeframe)
+
+    if groupedpair:
+        print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()],
+                       headers=("Pair", "Timeframe"),
+                       tablefmt='psql', stralign='right'))

freqtrade/data/history/idatahandler.py

@@ -13,6 +13,7 @@ from typing import List, Optional, Type
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
+from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.data.converter import (clean_ohlcv_dataframe,
                                       trades_remove_duplicates, trim_dataframe)
 from freqtrade.exchange import timeframe_to_seconds
@@ -28,6 +29,14 @@ class IDataHandler(ABC):
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
 
+    @abstractclassmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+
     @abstractclassmethod
     def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
         """

freqtrade/data/history/jsondatahandler.py

@@ -8,7 +8,8 @@ from pandas import DataFrame, read_json, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
+                                 ListPairsWithTimeframes)
 from freqtrade.data.converter import trades_dict_to_list
 
 from .idatahandler import IDataHandler, TradeList
@@ -21,6 +22,18 @@ class JsonDataHandler(IDataHandler):
     _use_zip = False
     _columns = DEFAULT_DATAFRAME_COLUMNS
 
+    @classmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+        _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name)
+                for p in datadir.glob(f"*.{cls._get_file_extension()}")]
+        return [(match[1].replace('_', '/'), match[2]) for match in _tmp
+                if match and len(match.groups()) > 1]
+
     @classmethod
     def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
         """

tests/commands/test_commands.py

@@ -6,12 +6,12 @@ import pytest
 
 from freqtrade.commands import (start_convert_data, start_create_userdir,
                                 start_download_data, start_hyperopt_list,
-                                start_hyperopt_show, start_list_exchanges,
+                                start_hyperopt_show, start_list_data,
-                                start_list_hyperopts, start_list_markets,
+                                start_list_exchanges, start_list_hyperopts,
-                                start_list_strategies, start_list_timeframes,
+                                start_list_markets, start_list_strategies,
-                                start_new_hyperopt, start_new_strategy,
+                                start_list_timeframes, start_new_hyperopt,
-                                start_show_trades, start_test_pairlist,
+                                start_new_strategy, start_show_trades,
-                                start_trading)
+                                start_test_pairlist, start_trading)
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
@@ -1043,6 +1043,40 @@ def test_convert_data_trades(mocker, testdatadir):
     assert trades_mock.call_args[1]['erase'] is False
 
 
+def test_start_list_data(testdatadir, capsys):
+    args = [
+        "list-data",
+        "--data-format-ohlcv",
+        "json",
+        "--datadir",
+        str(testdatadir),
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_list_data(pargs)
+    captured = capsys.readouterr()
+    assert "Found 16 pair / timeframe combinations." in captured.out
+    assert "\n|         Pair |       Timeframe |\n" in captured.out
+    assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m |\n" in captured.out
+
+    args = [
+        "list-data",
+        "--data-format-ohlcv",
+        "json",
+        "--pairs", "XRP/ETH",
+        "--datadir",
+        str(testdatadir),
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_list_data(pargs)
+    captured = capsys.readouterr()
+    assert "Found 2 pair / timeframe combinations." in captured.out
+    assert "\n|    Pair |   Timeframe |\n" in captured.out
+    assert "UNITTEST/BTC" not in captured.out
+    assert "\n| XRP/ETH |      1m, 5m |\n" in captured.out
+
+
 @pytest.mark.usefixtures("init_persistence")
 def test_show_trades(mocker, fee, capsys, caplog):
     mocker.patch("freqtrade.persistence.init")

tests/data/test_history.py

@@ -631,6 +631,20 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
     assert set(pairs) == {'UNITTEST/BTC'}
 
 
+def test_jsondatahandler_ohlcv_get_available_data(testdatadir):
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'),
+                              ('TRX/BTC', '5m'), ('LTC/BTC', '5m'), ('XMR/BTC', '5m'),
+                              ('ZEC/BTC', '5m'), ('UNITTEST/BTC', '1m'), ('ADA/BTC', '5m'),
+                              ('ETC/BTC', '5m'), ('NXT/BTC', '5m'), ('DASH/BTC', '5m'),
+                              ('XRP/ETH', '1m'), ('XRP/ETH', '5m'), ('UNITTEST/BTC', '30m'),
+                              ('UNITTEST/BTC', '8m')}
+
+    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir)
+    assert set(paircombs) == {('UNITTEST/BTC', '8m')}
+
+
 def test_jsondatahandler_trades_get_pairs(testdatadir):
     pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
     # Convert to set to avoid failures due to sorting