Merge pull request #5642 from freqtrade/add_trade_conversion_command

Add trade conversion command

docs/data-download.md

@@ -204,6 +204,61 @@ It'll also remove original jsongz data files (`--erase` parameter).
 freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
 ```
 
+### Sub-command trades to ohlcv
+
+When you need to use `--dl-trades` (kraken only) to download data, conversion of trades data to ohlcv data is the last step.
+This command will allow you to repeat this last step for additional timeframes without re-downloading the data.
+
+```
+usage: freqtrade trades-to-ohlcv [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                                 [-d PATH] [--userdir PATH]
+                                 [-p PAIRS [PAIRS ...]]
+                                 [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
+                                 [--exchange EXCHANGE]
+                                 [--data-format-ohlcv {json,jsongz,hdf5}]
+                                 [--data-format-trades {json,jsongz,hdf5}]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
+  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
+                        Specify which tickers to download. Space-separated
+                        list. Default: `1m 5m`.
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
+  --data-format-ohlcv {json,jsongz,hdf5}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `json`).
+  --data-format-trades {json,jsongz,hdf5}
+                        Storage format for downloaded trades data. (default:
+                        `jsongz`).
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+
+```
+
+#### Example trade-to-ohlcv conversion
+
+``` bash
+freqtrade trades-to-ohlcv --exchange kraken -t 5m 1h 1d --pairs BTC/EUR ETH/EUR
+```
+
 ### Sub-command list-data
 
 You can get a list of downloaded data using the `list-data` sub-command.

freqtrade/commands/__init__.py

@@ -8,8 +8,8 @@ Note: Be careful with file-scoped imports in these subfiles.
 """
 from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
-from freqtrade.commands.data_commands import (start_convert_data, start_download_data,
+from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
-                                              start_list_data)
+                                              start_download_data, start_list_data)
 from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
                                                 start_new_strategy)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show

freqtrade/commands/arguments.py

@@ -58,6 +58,8 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
+ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
+
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
 
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
@@ -91,7 +93,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
                     "list-markets", "list-pairs", "list-strategies", "list-data",
                     "hyperopt-list", "hyperopt-show",
-                    "plot-dataframe", "plot-profit", "show-trades"]
+                    "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"]
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
 
@@ -169,14 +171,14 @@ class Arguments:
         self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
         self._build_args(optionlist=['version'], parser=self.parser)
 
-        from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir,
+        from freqtrade.commands import (start_backtesting, start_convert_data, start_convert_trades,
-                                        start_download_data, start_edge, start_hyperopt,
+                                        start_create_userdir, start_download_data, start_edge,
-                                        start_hyperopt_list, start_hyperopt_show, start_install_ui,
+                                        start_hyperopt, start_hyperopt_list, start_hyperopt_show,
-                                        start_list_data, start_list_exchanges, start_list_markets,
+                                        start_install_ui, start_list_data, start_list_exchanges,
-                                        start_list_strategies, start_list_timeframes,
+                                        start_list_markets, start_list_strategies,
-                                        start_new_config, start_new_strategy, start_plot_dataframe,
+                                        start_list_timeframes, start_new_config, start_new_strategy,
-                                        start_plot_profit, start_show_trades, start_test_pairlist,
+                                        start_plot_dataframe, start_plot_profit, start_show_trades,
-                                        start_trading, start_webserver)
+                                        start_test_pairlist, start_trading, start_webserver)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added
@@ -236,6 +238,15 @@ class Arguments:
         convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
         self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
 
+        # Add trades-to-ohlcv subcommand
+        convert_trade_data_cmd = subparsers.add_parser(
+            'trades-to-ohlcv',
+            help='Convert trade data to OHLCV data.',
+            parents=[_common_parser],
+        )
+        convert_trade_data_cmd.set_defaults(func=start_convert_trades)
+        self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd)
+
         # Add list-data subcommand
         list_data_cmd = subparsers.add_parser(
             'list-data',

freqtrade/commands/cli_options.py

@@ -381,12 +381,12 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "dataformat_ohlcv": Arg(
         '--data-format-ohlcv',
-        help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).',
+        help='Storage format for downloaded candle (OHLCV) data. (default: `json`).',
         choices=constants.AVAILABLE_DATAHANDLERS,
     ),
     "dataformat_trades": Arg(
         '--data-format-trades',
-        help='Storage format for downloaded trades data. (default: `%(default)s`).',
+        help='Storage format for downloaded trades data. (default: `jsongz`).',
         choices=constants.AVAILABLE_DATAHANDLERS,
     ),
     "exchange": Arg(

freqtrade/commands/data_commands.py

@@ -89,6 +89,41 @@ def start_download_data(args: Dict[str, Any]) -> None:
                         f"on exchange {exchange.name}.")
 
 
+def start_convert_trades(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    timerange = TimeRange()
+
+    # Remove stake-currency to skip checks which are not relevant for datadownload
+    config['stake_currency'] = ''
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+    # Manual validations of relevant settings
+    if not config['exchange'].get('skip_pair_validation', False):
+        exchange.validate_pairs(config['pairs'])
+    expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
+
+    logger.info(f"About to Convert pairs: {expanded_pairs}, "
+                f"intervals: {config['timeframes']} to {config['datadir']}")
+
+    for timeframe in config['timeframes']:
+        exchange.validate_timeframes(timeframe)
+    # Convert downloaded trade data to different timeframes
+    convert_trades_to_ohlcv(
+        pairs=expanded_pairs, timeframes=config['timeframes'],
+        datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
+        data_format_ohlcv=config['dataformat_ohlcv'],
+        data_format_trades=config['dataformat_trades'],
+    )
+
+
 def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     Convert data from one format to another

tests/commands/test_commands.py

@@ -8,12 +8,12 @@ from zipfile import ZipFile
 import arrow
 import pytest
 
-from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data,
+from freqtrade.commands import (start_convert_data, start_convert_trades, start_create_userdir,
-                                start_hyperopt_list, start_hyperopt_show, start_install_ui,
+                                start_download_data, start_hyperopt_list, start_hyperopt_show,
-                                start_list_data, start_list_exchanges, start_list_markets,
+                                start_install_ui, start_list_data, start_list_exchanges,
-                                start_list_strategies, start_list_timeframes, start_new_strategy,
+                                start_list_markets, start_list_strategies, start_list_timeframes,
-                                start_show_trades, start_test_pairlist, start_trading,
+                                start_new_strategy, start_show_trades, start_test_pairlist,
-                                start_webserver)
+                                start_trading, start_webserver)
 from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui,
                                                 get_ui_download_url, read_ui_version)
 from freqtrade.configuration import setup_utils_configuration
@@ -759,6 +759,22 @@ def test_download_data_trades(mocker, caplog):
     assert convert_mock.call_count == 1
 
 
+def test_start_convert_trades(mocker, caplog):
+    convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
+                                MagicMock(return_value=[]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "trades-to-ohlcv",
+        "--exchange", "kraken",
+        "--pairs", "ETH/BTC", "XRP/BTC",
+    ]
+    start_convert_trades(get_args(args))
+    assert convert_mock.call_count == 1
+
+
 def test_start_list_strategies(mocker, caplog, capsys):
 
     args = [