From 422825ea1b57494716697e550884fd00b8bac80f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jul 2020 09:50:53 +0200 Subject: [PATCH 1/8] Add ohlcv_get_available_data to find available data --- freqtrade/data/history/idatahandler.py | 9 +++++++++ freqtrade/data/history/jsondatahandler.py | 14 +++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index d5d7c16db..e255710cb 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -13,6 +13,7 @@ from typing import List, Optional, Type from pandas import DataFrame from freqtrade.configuration import TimeRange +from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.converter import (clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe) from freqtrade.exchange import timeframe_to_seconds @@ -28,6 +29,14 @@ class IDataHandler(ABC): def __init__(self, datadir: Path) -> None: self._datadir = datadir + @abstractclassmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Pair + """ + @abstractclassmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 01320f129..79a848d07 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -8,7 +8,8 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, + ListPairsWithTimeframes) from freqtrade.data.converter import trades_dict_to_list from .idatahandler import IDataHandler, TradeList @@ -21,6 +22,17 @@ class JsonDataHandler(IDataHandler): _use_zip = False _columns = DEFAULT_DATAFRAME_COLUMNS + @classmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Pair + """ + _tmp = [re.search(r'^([a-zA-Z_]+)\-(\S+)(?=.json)', p.name) + for p in datadir.glob(f"*.{cls._get_file_extension()}")] + return [(match[1].replace('_', '/'), match[2]) for match in _tmp if len(match.groups()) > 1] + @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: """ From d4fc52d2d5b6e9bd6a5fe06ce902ea6460375db4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jul 2020 09:56:46 +0200 Subject: [PATCH 2/8] Add tests for ohlcv_get_available_data --- freqtrade/data/history/jsondatahandler.py | 5 +++-- tests/data/test_history.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 79a848d07..4ef35cf55 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -29,9 +29,10 @@ class JsonDataHandler(IDataHandler): :param datadir: Directory to search for ohlcv files :return: List of Pair """ - _tmp = [re.search(r'^([a-zA-Z_]+)\-(\S+)(?=.json)', p.name) + _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name) for p in datadir.glob(f"*.{cls._get_file_extension()}")] - return [(match[1].replace('_', '/'), match[2]) for match in _tmp if len(match.groups()) > 1] + return [(match[1].replace('_', '/'), match[2]) for match in _tmp + if match and len(match.groups()) > 1] @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index c2eb2d715..d84c212b1 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -631,6 +631,20 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir): assert set(pairs) == {'UNITTEST/BTC'} +def test_jsondatahandler_ohlcv_get_available_data(testdatadir): + paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) + # Convert to set to avoid failures due to sorting + assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'), + ('TRX/BTC', '5m'), ('LTC/BTC', '5m'), ('XMR/BTC', '5m'), + ('ZEC/BTC', '5m'), ('UNITTEST/BTC', '1m'), ('ADA/BTC', '5m'), + ('ETC/BTC', '5m'), ('NXT/BTC', '5m'), ('DASH/BTC', '5m'), + ('XRP/ETH', '1m'), ('XRP/ETH', '5m'), ('UNITTEST/BTC', '30m'), + ('UNITTEST/BTC', '8m')} + + paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir) + assert set(paircombs) == {('UNITTEST/BTC', '8m')} + + def test_jsondatahandler_trades_get_pairs(testdatadir): pairs = JsonGzDataHandler.trades_get_pairs(testdatadir) # Convert to set to avoid failures due to sorting From 02afde857d204f91d9170bb9bf22439b07e8c2cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jul 2020 09:57:00 +0200 Subject: [PATCH 3/8] Add list-data command --- freqtrade/commands/__init__.py | 3 ++- freqtrade/commands/arguments.py | 15 +++++++++++++-- freqtrade/commands/data_commands.py | 23 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 2d0c7733c..4ce3eb421 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -9,7 +9,8 @@ Note: Be careful with file-scoped imports in these subfiles. from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, - start_download_data) + start_download_data, + start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_new_hyperopt, start_new_strategy) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 72f2a02f0..a49d917a5 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -54,6 +54,8 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] +ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv"] + ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] @@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperopt_show_no_header"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", - "list-markets", "list-pairs", "list-strategies", + "list-markets", "list-pairs", "list-strategies", "list-data", "list-hyperopts", "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit", "show-trades"] @@ -159,7 +161,7 @@ class Arguments: self._build_args(optionlist=['version'], parser=self.parser) from freqtrade.commands import (start_create_userdir, start_convert_data, - start_download_data, + start_download_data, start_list_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, @@ -233,6 +235,15 @@ class Arguments: convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd) + # Add list-data subcommand + list_data_cmd = subparsers.add_parser( + 'list-data', + help='List downloaded data.', + parents=[_common_parser], + ) + list_data_cmd.set_defaults(func=start_list_data) + self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd) + # Add backtesting subcommand backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.', parents=[_common_parser, _strategy_parser]) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index fc3a49f1d..4a37bdc08 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -1,5 +1,6 @@ import logging import sys +from collections import defaultdict from typing import Any, Dict, List import arrow @@ -11,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_minutes from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -88,3 +90,24 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: convert_trades_format(config, convert_from=args['format_from'], convert_to=args['format_to'], erase=args['erase']) + + +def start_list_data(args: Dict[str, Any]) -> None: + """ + List available backtest data + """ + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + from freqtrade.data.history.idatahandler import get_datahandlerclass + from tabulate import tabulate + dhc = get_datahandlerclass(config['dataformat_ohlcv']) + paircombs = dhc.ohlcv_get_available_data(config['datadir']) + + print(f"Found {len(paircombs)} pair / timeframe combinations.") + groupedpair = defaultdict(list) + for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))): + groupedpair[pair].append(timeframe) + + print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()], + headers=("pairs", "timeframe"))) From 5bb81abce26b4e1bc44f7bc28cc0cf04397fea57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jul 2020 10:01:37 +0200 Subject: [PATCH 4/8] Add test for start_list_data --- tests/commands/test_commands.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 46350beff..9c741e102 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -6,12 +6,12 @@ import pytest from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data, start_hyperopt_list, - start_hyperopt_show, start_list_exchanges, - start_list_hyperopts, start_list_markets, - start_list_strategies, start_list_timeframes, - start_new_hyperopt, start_new_strategy, - start_show_trades, start_test_pairlist, - start_trading) + start_hyperopt_show, start_list_data, + start_list_exchanges, start_list_hyperopts, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_hyperopt, + start_new_strategy, start_show_trades, + start_test_pairlist, start_trading) from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode @@ -1043,6 +1043,23 @@ def test_convert_data_trades(mocker, testdatadir): assert trades_mock.call_args[1]['erase'] is False +def test_start_list_data(testdatadir, capsys): + args = [ + "list-data", + "--data-format-ohlcv", + "json", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found 16 pair / timeframe combinations." in captured.out + assert "\npairs timeframe\n" in captured.out + assert "\nUNITTEST/BTC 1m, 5m, 8m, 30m\n" in captured.out + + @pytest.mark.usefixtures("init_persistence") def test_show_trades(mocker, fee, capsys, caplog): mocker.patch("freqtrade.persistence.init") From 33c3990972495f1754744b98775f63e99a85aa0b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jul 2020 10:05:47 +0200 Subject: [PATCH 5/8] Add documentation for list-data command --- docs/data-download.md | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/data-download.md b/docs/data-download.md index 3fb775e69..7fbad0b6c 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -158,6 +158,55 @@ It'll also remove original jsongz data files (`--erase` parameter). freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase ``` +### Subcommand list-data + +You can get a list of downloaded data using the `list-data` subcommand. + +``` +usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] + [--userdir PATH] [--exchange EXCHANGE] + [--data-format-ohlcv {json,jsongz}] + +optional arguments: + -h, --help show this help message and exit + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + --data-format-ohlcv {json,jsongz} + Storage format for downloaded candle (OHLCV) data. + (default: `json`). + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + +#### Example list-data + +```bash +> freqtrade list-data --userdir ~/.freqtrade/user_data/ + +Found 33 pair / timeframe combinations. +pairs timeframe +---------- ----------------------------------------- +ADA/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d +ADA/ETH 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d +ETH/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d +ETH/USDT 5m, 15m, 30m, 1h, 2h, 4h +``` + ### Pairs file In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. From b035d9e2671b57cbb09ab340d432349b934182b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jul 2020 10:23:09 +0200 Subject: [PATCH 6/8] Update return type comment --- freqtrade/commands/data_commands.py | 5 +++-- freqtrade/data/history/idatahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 4a37bdc08..d3f70b9ec 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -99,9 +99,10 @@ def start_list_data(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - from freqtrade.data.history.idatahandler import get_datahandlerclass + from freqtrade.data.history.idatahandler import get_datahandler from tabulate import tabulate - dhc = get_datahandlerclass(config['dataformat_ohlcv']) + dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv']) + paircombs = dhc.ohlcv_get_available_data(config['datadir']) print(f"Found {len(paircombs)} pair / timeframe combinations.") diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index e255710cb..96d288e01 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -34,7 +34,7 @@ class IDataHandler(ABC): """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files - :return: List of Pair + :return: List of Tuples of (pair, timeframe) """ @abstractclassmethod diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 4ef35cf55..2e7c0f773 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -27,7 +27,7 @@ class JsonDataHandler(IDataHandler): """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files - :return: List of Pair + :return: List of Tuples of (pair, timeframe) """ _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name) for p in datadir.glob(f"*.{cls._get_file_extension()}")] From 62c55b18631398c7447b5eed355fa495f0a299af Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Jul 2020 06:55:34 +0200 Subject: [PATCH 7/8] Enhance formatting, Add pair filter --- docs/data-download.md | 5 ++++- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/data_commands.py | 6 +++++- tests/commands/test_commands.py | 21 +++++++++++++++++++-- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 7fbad0b6c..a2bbec837 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -166,6 +166,7 @@ You can get a list of downloaded data using the `list-data` subcommand. usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [--data-format-ohlcv {json,jsongz}] + [-p PAIRS [PAIRS ...]] optional arguments: -h, --help show this help message and exit @@ -174,6 +175,9 @@ optional arguments: --data-format-ohlcv {json,jsongz} Storage format for downloaded candle (OHLCV) data. (default: `json`). + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -190,7 +194,6 @@ Common arguments: Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. - ``` #### Example list-data diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index a49d917a5..e6f6f8167 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -54,7 +54,7 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] -ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv"] +ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index d3f70b9ec..13b796a1e 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -105,10 +105,14 @@ def start_list_data(args: Dict[str, Any]) -> None: paircombs = dhc.ohlcv_get_available_data(config['datadir']) + if args['pairs']: + paircombs = [comb for comb in paircombs if comb[0] in args['pairs']] + print(f"Found {len(paircombs)} pair / timeframe combinations.") groupedpair = defaultdict(list) for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))): groupedpair[pair].append(timeframe) print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()], - headers=("pairs", "timeframe"))) + headers=("Pair", "Timeframe"), + tablefmt='psql', stralign='right')) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 9c741e102..ffced956d 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1056,8 +1056,25 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 16 pair / timeframe combinations." in captured.out - assert "\npairs timeframe\n" in captured.out - assert "\nUNITTEST/BTC 1m, 5m, 8m, 30m\n" in captured.out + assert "\n| Pair | Timeframe |\n" in captured.out + assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m |\n" in captured.out + + args = [ + "list-data", + "--data-format-ohlcv", + "json", + "--pairs", "XRP/ETH", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found 2 pair / timeframe combinations." in captured.out + assert "\n| Pair | Timeframe |\n" in captured.out + assert "UNITTEST/BTC" not in captured.out + assert "\n| XRP/ETH | 1m, 5m |\n" in captured.out @pytest.mark.usefixtures("init_persistence") From 0228b63418bb35d680c9c4bfe8e0c2b492cf3b6a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Jul 2020 16:42:47 +0200 Subject: [PATCH 8/8] Don't print empty table --- freqtrade/commands/data_commands.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 13b796a1e..aa0b826b5 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -113,6 +113,7 @@ def start_list_data(args: Dict[str, Any]) -> None: for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))): groupedpair[pair].append(timeframe) - print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()], - headers=("Pair", "Timeframe"), - tablefmt='psql', stralign='right')) + if groupedpair: + print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()], + headers=("Pair", "Timeframe"), + tablefmt='psql', stralign='right'))