Merge pull request #3582 from freqtrade/data/list
List available backtesting data
This commit is contained in:
hroff-1902
GitHub
commit
0f4fc67b83
@ -158,6 +158,58 @@ It'll also remove original jsongz data files (`--erase` parameter).
|
||||
freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
|
||||
```
|
||||
|
||||
### Subcommand list-data
|
||||
|
||||
You can get a list of downloaded data using the `list-data` subcommand.
|
||||
|
||||
```
|
||||
usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
|
||||
[--userdir PATH] [--exchange EXCHANGE]
|
||||
[--data-format-ohlcv {json,jsongz}]
|
||||
[-p PAIRS [PAIRS ...]]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no
|
||||
config is provided.
|
||||
--data-format-ohlcv {json,jsongz}
|
||||
Storage format for downloaded candle (OHLCV) data.
|
||||
(default: `json`).
|
||||
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
|
||||
Show profits for only these pairs. Pairs are space-
|
||||
separated.
|
||||
|
||||
Common arguments:
|
||||
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).
|
||||
--logfile FILE Log to the file specified. Special values are:
|
||||
'syslog', 'journald'. See the documentation for more
|
||||
details.
|
||||
-V, --version show program's version number and exit
|
||||
-c PATH, --config PATH
|
||||
Specify configuration file (default:
|
||||
`userdir/config.json` or `config.json` whichever
|
||||
exists). Multiple --config options may be used. Can be
|
||||
set to `-` to read config from stdin.
|
||||
-d PATH, --datadir PATH
|
||||
Path to directory with historical backtesting data.
|
||||
--userdir PATH, --user-data-dir PATH
|
||||
Path to userdata directory.
|
||||
```
|
||||
|
||||
#### Example list-data
|
||||
|
||||
```bash
|
||||
> freqtrade list-data --userdir ~/.freqtrade/user_data/
|
||||
|
||||
Found 33 pair / timeframe combinations.
|
||||
pairs timeframe
|
||||
---------- -----------------------------------------
|
||||
ADA/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
|
||||
ADA/ETH 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
|
||||
ETH/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
|
||||
ETH/USDT 5m, 15m, 30m, 1h, 2h, 4h
|
||||
```
|
||||
|
||||
### Pairs file
|
||||
|
||||
In alternative to the whitelist from `config.json`, a `pairs.json` file can be used.
|
||||
|
||||
@ -9,7 +9,8 @@ Note: Be careful with file-scoped imports in these subfiles.
|
||||
from freqtrade.commands.arguments import Arguments
|
||||
from freqtrade.commands.build_config_commands import start_new_config
|
||||
from freqtrade.commands.data_commands import (start_convert_data,
|
||||
start_download_data)
|
||||
start_download_data,
|
||||
start_list_data)
|
||||
from freqtrade.commands.deploy_commands import (start_create_userdir,
|
||||
start_new_hyperopt,
|
||||
start_new_strategy)
|
||||
|
||||
@ -54,6 +54,8 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
|
||||
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
|
||||
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
|
||||
|
||||
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
|
||||
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
|
||||
"timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
|
||||
|
||||
@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
|
||||
"print_json", "hyperopt_show_no_header"]
|
||||
|
||||
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
|
||||
"list-markets", "list-pairs", "list-strategies",
|
||||
"list-markets", "list-pairs", "list-strategies", "list-data",
|
||||
"list-hyperopts", "hyperopt-list", "hyperopt-show",
|
||||
"plot-dataframe", "plot-profit", "show-trades"]
|
||||
|
||||
@ -159,7 +161,7 @@ class Arguments:
|
||||
self._build_args(optionlist=['version'], parser=self.parser)
|
||||
|
||||
from freqtrade.commands import (start_create_userdir, start_convert_data,
|
||||
start_download_data,
|
||||
start_download_data, start_list_data,
|
||||
start_hyperopt_list, start_hyperopt_show,
|
||||
start_list_exchanges, start_list_hyperopts,
|
||||
start_list_markets, start_list_strategies,
|
||||
@ -233,6 +235,15 @@ class Arguments:
|
||||
convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
|
||||
self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
|
||||
|
||||
# Add list-data subcommand
|
||||
list_data_cmd = subparsers.add_parser(
|
||||
'list-data',
|
||||
help='List downloaded data.',
|
||||
parents=[_common_parser],
|
||||
)
|
||||
list_data_cmd.set_defaults(func=start_list_data)
|
||||
self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
|
||||
|
||||
# Add backtesting subcommand
|
||||
backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
|
||||
parents=[_common_parser, _strategy_parser])
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import arrow
|
||||
@ -11,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv,
|
||||
refresh_backtest_ohlcv_data,
|
||||
refresh_backtest_trades_data)
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import timeframe_to_minutes
|
||||
from freqtrade.resolvers import ExchangeResolver
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
@ -88,3 +90,30 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
|
||||
convert_trades_format(config,
|
||||
convert_from=args['format_from'], convert_to=args['format_to'],
|
||||
erase=args['erase'])
|
||||
|
||||
|
||||
def start_list_data(args: Dict[str, Any]) -> None:
|
||||
"""
|
||||
List available backtest data
|
||||
"""
|
||||
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
|
||||
|
||||
from freqtrade.data.history.idatahandler import get_datahandler
|
||||
from tabulate import tabulate
|
||||
dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
|
||||
|
||||
paircombs = dhc.ohlcv_get_available_data(config['datadir'])
|
||||
|
||||
if args['pairs']:
|
||||
paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
|
||||
|
||||
print(f"Found {len(paircombs)} pair / timeframe combinations.")
|
||||
groupedpair = defaultdict(list)
|
||||
for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))):
|
||||
groupedpair[pair].append(timeframe)
|
||||
|
||||
if groupedpair:
|
||||
print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()],
|
||||
headers=("Pair", "Timeframe"),
|
||||
tablefmt='psql', stralign='right'))
|
||||
|
||||
@ -13,6 +13,7 @@ from typing import List, Optional, Type
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.constants import ListPairsWithTimeframes
|
||||
from freqtrade.data.converter import (clean_ohlcv_dataframe,
|
||||
trades_remove_duplicates, trim_dataframe)
|
||||
from freqtrade.exchange import timeframe_to_seconds
|
||||
@ -28,6 +29,14 @@ class IDataHandler(ABC):
|
||||
def __init__(self, datadir: Path) -> None:
|
||||
self._datadir = datadir
|
||||
|
||||
@abstractclassmethod
|
||||
def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
|
||||
"""
|
||||
Returns a list of all pairs with ohlcv data available in this datadir
|
||||
:param datadir: Directory to search for ohlcv files
|
||||
:return: List of Tuples of (pair, timeframe)
|
||||
"""
|
||||
|
||||
@abstractclassmethod
|
||||
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
|
||||
"""
|
||||
|
||||
@ -8,7 +8,8 @@ from pandas import DataFrame, read_json, to_datetime
|
||||
|
||||
from freqtrade import misc
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
|
||||
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
|
||||
ListPairsWithTimeframes)
|
||||
from freqtrade.data.converter import trades_dict_to_list
|
||||
|
||||
from .idatahandler import IDataHandler, TradeList
|
||||
@ -21,6 +22,18 @@ class JsonDataHandler(IDataHandler):
|
||||
_use_zip = False
|
||||
_columns = DEFAULT_DATAFRAME_COLUMNS
|
||||
|
||||
@classmethod
|
||||
def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
|
||||
"""
|
||||
Returns a list of all pairs with ohlcv data available in this datadir
|
||||
:param datadir: Directory to search for ohlcv files
|
||||
:return: List of Tuples of (pair, timeframe)
|
||||
"""
|
||||
_tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name)
|
||||
for p in datadir.glob(f"*.{cls._get_file_extension()}")]
|
||||
return [(match[1].replace('_', '/'), match[2]) for match in _tmp
|
||||
if match and len(match.groups()) > 1]
|
||||
|
||||
@classmethod
|
||||
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
|
||||
"""
|
||||
|
||||
@ -6,12 +6,12 @@ import pytest
|
||||
|
||||
from freqtrade.commands import (start_convert_data, start_create_userdir,
|
||||
start_download_data, start_hyperopt_list,
|
||||
start_hyperopt_show, start_list_exchanges,
|
||||
start_list_hyperopts, start_list_markets,
|
||||
start_list_strategies, start_list_timeframes,
|
||||
start_new_hyperopt, start_new_strategy,
|
||||
start_show_trades, start_test_pairlist,
|
||||
start_trading)
|
||||
start_hyperopt_show, start_list_data,
|
||||
start_list_exchanges, start_list_hyperopts,
|
||||
start_list_markets, start_list_strategies,
|
||||
start_list_timeframes, start_new_hyperopt,
|
||||
start_new_strategy, start_show_trades,
|
||||
start_test_pairlist, start_trading)
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.state import RunMode
|
||||
@ -1043,6 +1043,40 @@ def test_convert_data_trades(mocker, testdatadir):
|
||||
assert trades_mock.call_args[1]['erase'] is False
|
||||
|
||||
|
||||
def test_start_list_data(testdatadir, capsys):
|
||||
args = [
|
||||
"list-data",
|
||||
"--data-format-ohlcv",
|
||||
"json",
|
||||
"--datadir",
|
||||
str(testdatadir),
|
||||
]
|
||||
pargs = get_args(args)
|
||||
pargs['config'] = None
|
||||
start_list_data(pargs)
|
||||
captured = capsys.readouterr()
|
||||
assert "Found 16 pair / timeframe combinations." in captured.out
|
||||
assert "\n| Pair | Timeframe |\n" in captured.out
|
||||
assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m |\n" in captured.out
|
||||
|
||||
args = [
|
||||
"list-data",
|
||||
"--data-format-ohlcv",
|
||||
"json",
|
||||
"--pairs", "XRP/ETH",
|
||||
"--datadir",
|
||||
str(testdatadir),
|
||||
]
|
||||
pargs = get_args(args)
|
||||
pargs['config'] = None
|
||||
start_list_data(pargs)
|
||||
captured = capsys.readouterr()
|
||||
assert "Found 2 pair / timeframe combinations." in captured.out
|
||||
assert "\n| Pair | Timeframe |\n" in captured.out
|
||||
assert "UNITTEST/BTC" not in captured.out
|
||||
assert "\n| XRP/ETH | 1m, 5m |\n" in captured.out
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_show_trades(mocker, fee, capsys, caplog):
|
||||
mocker.patch("freqtrade.persistence.init")
|
||||
|
||||
@ -631,6 +631,20 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
|
||||
assert set(pairs) == {'UNITTEST/BTC'}
|
||||
|
||||
|
||||
def test_jsondatahandler_ohlcv_get_available_data(testdatadir):
|
||||
paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir)
|
||||
# Convert to set to avoid failures due to sorting
|
||||
assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'),
|
||||
('TRX/BTC', '5m'), ('LTC/BTC', '5m'), ('XMR/BTC', '5m'),
|
||||
('ZEC/BTC', '5m'), ('UNITTEST/BTC', '1m'), ('ADA/BTC', '5m'),
|
||||
('ETC/BTC', '5m'), ('NXT/BTC', '5m'), ('DASH/BTC', '5m'),
|
||||
('XRP/ETH', '1m'), ('XRP/ETH', '5m'), ('UNITTEST/BTC', '30m'),
|
||||
('UNITTEST/BTC', '8m')}
|
||||
|
||||
paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir)
|
||||
assert set(paircombs) == {('UNITTEST/BTC', '8m')}
|
||||
|
||||
|
||||
def test_jsondatahandler_trades_get_pairs(testdatadir):
|
||||
pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
|
||||
# Convert to set to avoid failures due to sorting
|
||||
|
||||
Loading…
Reference in New Issue
Block a user