From 35857b3ddea9c4a73332a38677db533cacd8e55b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Sep 2020 09:10:34 +0200 Subject: [PATCH 1/3] Datetime should support --timerange too --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/data_commands.py | 6 ++++ tests/commands/test_commands.py | 45 +++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 7268c3c8f..498c73fb3 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -57,7 +57,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", - "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] + "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades", "timerange"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index da1eb0cf5..956a8693e 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -25,11 +25,17 @@ def start_download_data(args: Dict[str, Any]) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + if 'days' in config and 'timerange' in config: + raise OperationalException("--days and --timerange are mutually exclusive. " + "You can only specify one or the other.") timerange = TimeRange() if 'days' in config: time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') + if 'timerange' in config: + timerange = timerange.parse_timerange(config['timerange']) + if 'pairs' not in config: raise OperationalException( "Downloading data requires a list of pairs. " diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 26875fb7f..99b7139b0 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -2,6 +2,7 @@ import re from pathlib import Path from unittest.mock import MagicMock, PropertyMock +import arrow import pytest from freqtrade.commands import (start_convert_data, start_create_userdir, @@ -552,6 +553,50 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): assert dl_mock.call_count == 1 +def test_download_data_timerange(mocker, caplog, markets): + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + "--timerange", "20200101-" + ] + with pytest.raises(OperationalException, + match=r"--days and --timerange are mutually.*"): + start_download_data(get_args(args)) + assert dl_mock.call_count == 0 + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + start_download_data(get_args(args)) + assert dl_mock.call_count == 1 + # 20days ago + days_ago = arrow.get(arrow.utcnow().shift(days=-20).date()).timestamp + assert dl_mock.call_args_list[0][1]['timerange'].startts == days_ago + + dl_mock.reset_mock() + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--timerange", "20200101-" + ] + start_download_data(get_args(args)) + assert dl_mock.call_count == 1 + + assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow(2020, 1, 1).timestamp + + def test_download_data_no_markets(mocker, caplog): dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) From 2f6b00555a09da306f321a2effef242e958a60da Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Sep 2020 09:13:43 +0200 Subject: [PATCH 2/3] Document support for --timerange in download-data --- docs/data-download.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/data-download.md b/docs/data-download.md index 0b22ec9ce..932b84557 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -8,9 +8,11 @@ If no additional parameter is specified, freqtrade will download data for `"1m"` Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. +You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101`). For incremental downloads, the relative approach should be used. + !!! Tip "Tip: Updating existing data" If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. - Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. ### Usage @@ -24,6 +26,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--erase] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] + [--timerange TIMERANGE] optional arguments: -h, --help show this help message and exit @@ -48,6 +51,8 @@ optional arguments: --data-format-trades {json,jsongz,hdf5} Storage format for downloaded trades data. (default: `jsongz`). + --timerange TIMERANGE + Specify what timerange of data to use. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -286,6 +291,7 @@ This will download historical candle (OHLCV) data for all the currency pairs you - To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). +- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. From bf95fe2e5cee112fddf83e8d9b09cfa5d3e9e916 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Sep 2020 11:33:55 +0200 Subject: [PATCH 3/3] have the 2 timerange arguments next to each other --- docs/data-download.md | 9 ++++----- freqtrade/commands/arguments.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 932b84557..9065bb050 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -20,13 +20,12 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`- usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] [--pairs-file FILE] - [--days INT] [--dl-trades] - [--exchange EXCHANGE] + [--days INT] [--timerange TIMERANGE] + [--dl-trades] [--exchange EXCHANGE] [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] [--erase] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] - [--timerange TIMERANGE] optional arguments: -h, --help show this help message and exit @@ -35,6 +34,8 @@ optional arguments: separated. --pairs-file FILE File containing a list of pairs to download. --days INT Download data for given number of days. + --timerange TIMERANGE + Specify what timerange of data to use. --dl-trades Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as --timeframes/-t. @@ -51,8 +52,6 @@ optional arguments: --data-format-trades {json,jsongz,hdf5} Storage format for downloaded trades data. (default: `jsongz`). - --timerange TIMERANGE - Specify what timerange of data to use. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 498c73fb3..b61a4933e 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -56,8 +56,8 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] -ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", - "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades", "timerange"] +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange", + "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename",