Merge pull request #5642 from freqtrade/add_trade_conversion_command
Add trade conversion command
This commit is contained in:
Matthias
GitHub
commit
99e3450d30
@ -204,6 +204,61 @@ It'll also remove original jsongz data files (`--erase` parameter).
|
||||
freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
|
||||
```
|
||||
|
||||
### Sub-command trades to ohlcv
|
||||
|
||||
When you need to use `--dl-trades` (kraken only) to download data, conversion of trades data to ohlcv data is the last step.
|
||||
This command will allow you to repeat this last step for additional timeframes without re-downloading the data.
|
||||
|
||||
```
|
||||
usage: freqtrade trades-to-ohlcv [-h] [-v] [--logfile FILE] [-V] [-c PATH]
|
||||
[-d PATH] [--userdir PATH]
|
||||
[-p PAIRS [PAIRS ...]]
|
||||
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
|
||||
[--exchange EXCHANGE]
|
||||
[--data-format-ohlcv {json,jsongz,hdf5}]
|
||||
[--data-format-trades {json,jsongz,hdf5}]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
|
||||
Limit command to these pairs. Pairs are space-
|
||||
separated.
|
||||
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
|
||||
Specify which tickers to download. Space-separated
|
||||
list. Default: `1m 5m`.
|
||||
--exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no
|
||||
config is provided.
|
||||
--data-format-ohlcv {json,jsongz,hdf5}
|
||||
Storage format for downloaded candle (OHLCV) data.
|
||||
(default: `json`).
|
||||
--data-format-trades {json,jsongz,hdf5}
|
||||
Storage format for downloaded trades data. (default:
|
||||
`jsongz`).
|
||||
|
||||
Common arguments:
|
||||
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).
|
||||
--logfile FILE Log to the file specified. Special values are:
|
||||
'syslog', 'journald'. See the documentation for more
|
||||
details.
|
||||
-V, --version show program's version number and exit
|
||||
-c PATH, --config PATH
|
||||
Specify configuration file (default:
|
||||
`userdir/config.json` or `config.json` whichever
|
||||
exists). Multiple --config options may be used. Can be
|
||||
set to `-` to read config from stdin.
|
||||
-d PATH, --datadir PATH
|
||||
Path to directory with historical backtesting data.
|
||||
--userdir PATH, --user-data-dir PATH
|
||||
Path to userdata directory.
|
||||
|
||||
```
|
||||
|
||||
#### Example trade-to-ohlcv conversion
|
||||
|
||||
``` bash
|
||||
freqtrade trades-to-ohlcv --exchange kraken -t 5m 1h 1d --pairs BTC/EUR ETH/EUR
|
||||
```
|
||||
|
||||
### Sub-command list-data
|
||||
|
||||
You can get a list of downloaded data using the `list-data` sub-command.
|
||||
|
||||
@ -8,8 +8,8 @@ Note: Be careful with file-scoped imports in these subfiles.
|
||||
"""
|
||||
from freqtrade.commands.arguments import Arguments
|
||||
from freqtrade.commands.build_config_commands import start_new_config
|
||||
from freqtrade.commands.data_commands import (start_convert_data, start_download_data,
|
||||
start_list_data)
|
||||
from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
|
||||
start_download_data, start_list_data)
|
||||
from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
|
||||
start_new_strategy)
|
||||
from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
|
||||
|
||||
@ -58,6 +58,8 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
|
||||
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
|
||||
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
|
||||
|
||||
ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
|
||||
|
||||
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
|
||||
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
|
||||
@ -91,7 +93,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
|
||||
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
|
||||
"list-markets", "list-pairs", "list-strategies", "list-data",
|
||||
"hyperopt-list", "hyperopt-show",
|
||||
"plot-dataframe", "plot-profit", "show-trades"]
|
||||
"plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"]
|
||||
|
||||
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
|
||||
|
||||
@ -169,14 +171,14 @@ class Arguments:
|
||||
self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
|
||||
self._build_args(optionlist=['version'], parser=self.parser)
|
||||
|
||||
from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir,
|
||||
start_download_data, start_edge, start_hyperopt,
|
||||
start_hyperopt_list, start_hyperopt_show, start_install_ui,
|
||||
start_list_data, start_list_exchanges, start_list_markets,
|
||||
start_list_strategies, start_list_timeframes,
|
||||
start_new_config, start_new_strategy, start_plot_dataframe,
|
||||
start_plot_profit, start_show_trades, start_test_pairlist,
|
||||
start_trading, start_webserver)
|
||||
from freqtrade.commands import (start_backtesting, start_convert_data, start_convert_trades,
|
||||
start_create_userdir, start_download_data, start_edge,
|
||||
start_hyperopt, start_hyperopt_list, start_hyperopt_show,
|
||||
start_install_ui, start_list_data, start_list_exchanges,
|
||||
start_list_markets, start_list_strategies,
|
||||
start_list_timeframes, start_new_config, start_new_strategy,
|
||||
start_plot_dataframe, start_plot_profit, start_show_trades,
|
||||
start_test_pairlist, start_trading, start_webserver)
|
||||
|
||||
subparsers = self.parser.add_subparsers(dest='command',
|
||||
# Use custom message when no subhandler is added
|
||||
@ -236,6 +238,15 @@ class Arguments:
|
||||
convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
|
||||
self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
|
||||
|
||||
# Add trades-to-ohlcv subcommand
|
||||
convert_trade_data_cmd = subparsers.add_parser(
|
||||
'trades-to-ohlcv',
|
||||
help='Convert trade data to OHLCV data.',
|
||||
parents=[_common_parser],
|
||||
)
|
||||
convert_trade_data_cmd.set_defaults(func=start_convert_trades)
|
||||
self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd)
|
||||
|
||||
# Add list-data subcommand
|
||||
list_data_cmd = subparsers.add_parser(
|
||||
'list-data',
|
||||
|
||||
@ -381,12 +381,12 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
),
|
||||
"dataformat_ohlcv": Arg(
|
||||
'--data-format-ohlcv',
|
||||
help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).',
|
||||
help='Storage format for downloaded candle (OHLCV) data. (default: `json`).',
|
||||
choices=constants.AVAILABLE_DATAHANDLERS,
|
||||
),
|
||||
"dataformat_trades": Arg(
|
||||
'--data-format-trades',
|
||||
help='Storage format for downloaded trades data. (default: `%(default)s`).',
|
||||
help='Storage format for downloaded trades data. (default: `jsongz`).',
|
||||
choices=constants.AVAILABLE_DATAHANDLERS,
|
||||
),
|
||||
"exchange": Arg(
|
||||
|
||||
@ -89,6 +89,41 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
f"on exchange {exchange.name}.")
|
||||
|
||||
|
||||
def start_convert_trades(args: Dict[str, Any]) -> None:
|
||||
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
|
||||
|
||||
timerange = TimeRange()
|
||||
|
||||
# Remove stake-currency to skip checks which are not relevant for datadownload
|
||||
config['stake_currency'] = ''
|
||||
|
||||
if 'pairs' not in config:
|
||||
raise OperationalException(
|
||||
"Downloading data requires a list of pairs. "
|
||||
"Please check the documentation on how to configure this.")
|
||||
|
||||
# Init exchange
|
||||
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
|
||||
# Manual validations of relevant settings
|
||||
if not config['exchange'].get('skip_pair_validation', False):
|
||||
exchange.validate_pairs(config['pairs'])
|
||||
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
|
||||
|
||||
logger.info(f"About to Convert pairs: {expanded_pairs}, "
|
||||
f"intervals: {config['timeframes']} to {config['datadir']}")
|
||||
|
||||
for timeframe in config['timeframes']:
|
||||
exchange.validate_timeframes(timeframe)
|
||||
# Convert downloaded trade data to different timeframes
|
||||
convert_trades_to_ohlcv(
|
||||
pairs=expanded_pairs, timeframes=config['timeframes'],
|
||||
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format_ohlcv=config['dataformat_ohlcv'],
|
||||
data_format_trades=config['dataformat_trades'],
|
||||
)
|
||||
|
||||
|
||||
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
|
||||
"""
|
||||
Convert data from one format to another
|
||||
|
||||
@ -8,12 +8,12 @@ from zipfile import ZipFile
|
||||
import arrow
|
||||
import pytest
|
||||
|
||||
from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data,
|
||||
start_hyperopt_list, start_hyperopt_show, start_install_ui,
|
||||
start_list_data, start_list_exchanges, start_list_markets,
|
||||
start_list_strategies, start_list_timeframes, start_new_strategy,
|
||||
start_show_trades, start_test_pairlist, start_trading,
|
||||
start_webserver)
|
||||
from freqtrade.commands import (start_convert_data, start_convert_trades, start_create_userdir,
|
||||
start_download_data, start_hyperopt_list, start_hyperopt_show,
|
||||
start_install_ui, start_list_data, start_list_exchanges,
|
||||
start_list_markets, start_list_strategies, start_list_timeframes,
|
||||
start_new_strategy, start_show_trades, start_test_pairlist,
|
||||
start_trading, start_webserver)
|
||||
from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui,
|
||||
get_ui_download_url, read_ui_version)
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
@ -759,6 +759,22 @@ def test_download_data_trades(mocker, caplog):
|
||||
assert convert_mock.call_count == 1
|
||||
|
||||
|
||||
def test_start_convert_trades(mocker, caplog):
|
||||
convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
|
||||
MagicMock(return_value=[]))
|
||||
patch_exchange(mocker)
|
||||
mocker.patch(
|
||||
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
|
||||
)
|
||||
args = [
|
||||
"trades-to-ohlcv",
|
||||
"--exchange", "kraken",
|
||||
"--pairs", "ETH/BTC", "XRP/BTC",
|
||||
]
|
||||
start_convert_trades(get_args(args))
|
||||
assert convert_mock.call_count == 1
|
||||
|
||||
|
||||
def test_start_list_strategies(mocker, caplog, capsys):
|
||||
|
||||
args = [
|
||||
|
||||
Loading…
Reference in New Issue
Block a user