From 91886120a7d17071a4663cd39c36d378b7d563c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:37:10 +0200 Subject: [PATCH 01/17] use nargs for --pairs argument --- freqtrade/configuration/cli_options.py | 6 ++++-- freqtrade/constants.py | 1 - freqtrade/plot/plotting.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 84686d1e6..d39013737 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -254,7 +254,8 @@ AVAILABLE_CLI_OPTIONS = { # Script options "pairs": Arg( '-p', '--pairs', - help='Show profits for only these pairs. Pairs are comma-separated.', + help='Show profits for only these pairs. Pairs are space-separated.', + nargs='+', ), # Download data "pairs_file": Arg( @@ -276,9 +277,10 @@ AVAILABLE_CLI_OPTIONS = { "timeframes": Arg( '-t', '--timeframes', help=f'Specify which tickers to download. Space-separated list. ' - f'Default: `{constants.DEFAULT_DOWNLOAD_TICKER_INTERVALS}`.', + f'Default: `1m 5m`.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w'], + default=['1m', '5m'], nargs='+', ), "erase": Arg( diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9b73adcfe..fbf44dec8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -23,7 +23,6 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] DRY_RUN_WALLET = 999.9 -DEFAULT_DOWNLOAD_TICKER_INTERVALS = '1m 5m' TICKER_INTERVALS = [ '1m', '3m', '5m', '15m', '30m', diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 947b3003c..e6da581a4 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -37,7 +37,7 @@ def init_plotscript(config): strategy = StrategyResolver(config).strategy if "pairs" in config: - pairs = config["pairs"].split(',') + pairs = config["pairs"] else: pairs = config["exchange"]["pair_whitelist"] From 05deb9e09bdccd0c19904ed2687cc6bd8f2bf29f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:42:44 +0200 Subject: [PATCH 02/17] Migrate download-script logic to utils.py --- freqtrade/configuration/arguments.py | 19 ++++++-- freqtrade/tests/test_arguments.py | 18 ++++---- freqtrade/utils.py | 66 ++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 16 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 926d02f8f..8fa16318a 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,7 +30,7 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_EXCHANGES = ["print_one_column"] -ARGS_DOWNLOADER = ARGS_COMMON + ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] +ARGS_DOWNLOADER = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", @@ -40,6 +40,8 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"]) +NO_CONF_REQURIED = ["start_download_data"] + class Arguments(object): """ @@ -75,7 +77,10 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if not self._no_default_config and parsed_arg.config is None: + # Allow no-config for certain commands (like downloading / plotting) + if (not self._no_default_config and parsed_arg.config is None + and not (hasattr(parsed_arg, 'func') + and parsed_arg.func.__name__ in NO_CONF_REQURIED)): parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -93,7 +98,7 @@ class Arguments(object): :return: None """ from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import start_list_exchanges + from freqtrade.utils import start_download_data, start_list_exchanges subparsers = self.parser.add_subparsers(dest='subparser') @@ -119,3 +124,11 @@ class Arguments(object): ) list_exchanges_cmd.set_defaults(func=start_list_exchanges) self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd) + + # Add download-data subcommand + download_data_cmd = subparsers.add_parser( + 'download-data', + help='Download backtesting data.' + ) + download_data_cmd.set_defaults(func=start_download_data) + self._build_args(optionlist=ARGS_DOWNLOADER, parser=download_data_cmd) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 2cb7ff6d7..31ab9dea8 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -50,10 +50,10 @@ def test_parse_args_verbose() -> None: def test_common_scripts_options() -> None: - arguments = Arguments(['-p', 'ETH/BTC'], '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() - assert args.pairs == 'ETH/BTC' + args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC'], '').get_parsed_arg() + + assert args.pairs == ['ETH/BTC', 'XRP/BTC'] + assert hasattr(args, "func") def test_parse_args_version() -> None: @@ -135,14 +135,14 @@ def test_parse_args_hyperopt_custom() -> None: def test_download_data_options() -> None: args = [ - '--pairs-file', 'file_with_pairs', '--datadir', 'datadir/directory', + 'download-data', + '--pairs-file', 'file_with_pairs', '--days', '30', '--exchange', 'binance' ] - arguments = Arguments(args, '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() + args = Arguments(args, '').get_parsed_arg() + assert args.pairs_file == 'file_with_pairs' assert args.datadir == 'datadir/directory' assert args.days == 30 @@ -162,7 +162,7 @@ def test_plot_dataframe_options() -> None: assert pargs.indicators1 == "sma10,sma100" assert pargs.indicators2 == "macd,fastd,fastk" assert pargs.plot_limit == 30 - assert pargs.pairs == "UNITTEST/BTC" + assert pargs.pairs == ["UNITTEST/BTC"] def test_check_int_positive() -> None: diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d550ef43c..d2770ba1a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -1,11 +1,16 @@ import logging +import sys from argparse import Namespace +from pathlib import Path from typing import Any, Dict -from freqtrade.configuration import Configuration -from freqtrade.exchange import available_exchanges -from freqtrade.state import RunMode +import arrow +from freqtrade.configuration import Configuration, TimeRange +from freqtrade.data.history import download_pair_history +from freqtrade.exchange import available_exchanges +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -17,7 +22,7 @@ def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any :return: Configuration """ configuration = Configuration(args, method) - config = configuration.load_config() + config = configuration.get_config() config['exchange']['dry_run'] = True # Ensure we do not use Exchange credentials @@ -39,3 +44,56 @@ def start_list_exchanges(args: Namespace) -> None: else: print(f"Exchanges supported by ccxt and available for Freqtrade: " f"{', '.join(available_exchanges())}") + + +def start_download_data(args: Namespace) -> None: + """ + Download data based + """ + config = setup_utils_configuration(args, RunMode.OTHER) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + dl_path = Path(config['datadir']) + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {dl_path}') + + pairs_not_available = [] + + try: + # Init exchange + exchange = ExchangeResolver(config['exchange']['name'], config).exchange + + for pair in config["pairs"]: + if pair not in exchange._api.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in config["timeframes"]: + pair_print = pair.replace('/', '_') + filename = f'{pair_print}-{ticker_interval}.json' + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info( + f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") + + # configuration.resolve_pairs_list() + print(config) From 8655e521d7c64812ffda7f69e95e4959a6fcf6f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:53:46 +0200 Subject: [PATCH 03/17] Adapt some tests --- freqtrade/tests/test_arguments.py | 2 +- freqtrade/tests/test_main.py | 4 +++- freqtrade/tests/test_plotting.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 31ab9dea8..601f41e63 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -4,7 +4,7 @@ import argparse import pytest from freqtrade.configuration import Arguments -from freqtrade.configuration.arguments import ARGS_DOWNLOADER, ARGS_PLOT_DATAFRAME +from freqtrade.configuration.arguments import ARGS_PLOT_DATAFRAME from freqtrade.configuration.cli_options import check_int_positive diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index d8ec532b0..409025a3c 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring from copy import deepcopy -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import pytest @@ -21,6 +21,7 @@ def test_parse_args_backtesting(mocker) -> None: further argument parsing is done in test_arguments.py """ backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) + backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting with pytest.raises(SystemExit): main(['backtesting']) @@ -36,6 +37,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) + hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt with pytest.raises(SystemExit): main(['hyperopt']) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index cd72160f8..94d40ab84 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -50,7 +50,7 @@ def test_init_plotscript(default_conf, mocker): assert "pairs" in ret assert "strategy" in ret - default_conf['pairs'] = "POWR/BTC,XLM/BTC" + default_conf['pairs'] = ["POWR/BTC", "XLM/BTC"] ret = init_plotscript(default_conf) assert "tickers" in ret assert "POWR/BTC" in ret["tickers"] From 3c15e3ebddd07d14ba085f4d226d5bea2d7a97a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:56:38 +0200 Subject: [PATCH 04/17] Default load minimal config --- freqtrade/configuration/configuration.py | 13 +++++++++++++ freqtrade/constants.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c95246fc0..e24ace34b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -52,6 +52,9 @@ class Configuration(object): # Keep this method as staticmethod, so it can be used from interactive environments config: Dict[str, Any] = {} + if not files: + return constants.MINIMAL_CONFIG + # We expect here a list of config filenames for path in files: logger.info(f'Using config: {path} ...') @@ -276,6 +279,16 @@ class Configuration(object): self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='timeframes', + logstring='timeframes --timeframes: {}') + + self._args_to_config(config, argname='days', + logstring='Detected --days: {}') + + if "exchange" in self.args and self.args.exchange: + config['exchange']['name'] = self.args.exchange + logger.info(f"Using exchange {config['exchange']['name']}") + def _process_runmode(self, config: Dict[str, Any]) -> None: if not self.runmode: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index fbf44dec8..b73a723eb 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -38,6 +38,20 @@ SUPPORTED_FIAT = [ "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" ] +MINIMAL_CONFIG = { + 'stake_currency': '', + 'dry_run': True, + 'exchange': { + 'name': '', + 'key': '', + 'secret': '', + 'pair_whitelist': [], + 'ccxt_async_config': { + 'enableRateLimit': True, + } + } +} + # Required json-schema for user specified config CONF_SCHEMA = { 'type': 'object', From 4e308a1a3e4aef76c809e53f1d60953399fb8655 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:56:57 +0200 Subject: [PATCH 05/17] Resolve pairlist in configuration --- freqtrade/configuration/configuration.py | 37 ++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e24ace34b..ed12b6501 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -90,6 +90,11 @@ class Configuration(object): self._process_runmode(config) + # Check if the exchange set by the user is supported + check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) + + self._resolve_pairs_list(config) + return config def _process_logging_options(self, config: Dict[str, Any]) -> None: @@ -150,9 +155,6 @@ class Configuration(object): if 'sd_notify' in self.args and self.args.sd_notify: config['internals'].update({'sd_notify': True}) - # Check if the exchange set by the user is supported - check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) - def _process_datadir_options(self, config: Dict[str, Any]) -> None: """ Extract information for sys.argv and load datadir configuration: @@ -348,3 +350,32 @@ class Configuration(object): logger.info(logstring.format(config[argname])) if deprecated_msg: warnings.warn(f"DEPRECATED: {deprecated_msg}", DeprecationWarning) + + def _resolve_pairs_list(self, config: Dict[str, Any]) -> None: + """ + Helper for download script. + Takes first found: + * -p (pairs argument) + * --pairs-file + * whitelist from config + """ + + if "pairs" in self.args and self.args.pairs: + return + + if "pairs_file" in self.args and self.args.pairs_file: + pairs_file = self.args.pairs_file + logger.info(f'Reading pairs file "{pairs_file}".') + # Download pairs from the pairs file if no config is specified + # or if pairs file is specified explicitely + if not pairs_file.exists(): + OperationalException(f'No pairs file found with path "{pairs_file}".') + + # with pairs_file.open() as file: + # pairs = list(set(json.load(file))) + + # pairs.sort() + + if "config" in self.args: + logger.info("Using pairlist from configuration.") + config['pairs'] = config.get('exchange', {}).get('pair_whitelist') From 219d0b7fb016f51f90f5778941d6aba72f907910 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:27:33 +0200 Subject: [PATCH 06/17] Adjust documentation to removed download-script --- docs/backtesting.md | 67 +++++++++++++----------- freqtrade/configuration/configuration.py | 3 ++ freqtrade/data/history.py | 2 +- freqtrade/tests/data/test_history.py | 4 +- freqtrade/utils.py | 4 +- 5 files changed, 43 insertions(+), 37 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 7e9f7ff53..f666c5b49 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -3,9 +3,43 @@ This page explains how to validate your strategy performance by using Backtesting. +## Getting data for backtesting / hyperopt + +To download backtesting data (candles / OHLCV), we recommend using the `freqtrade download-data` command. + +If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. +Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. + +Alternatively, a `pairs.json` file can be used. + +If you are using Binance for example: + +- create a directory `user_data/data/binance` and copy `pairs.json` in that directory. +- update the `pairs.json` to contain the currency pairs you are interested in. + +```bash +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance +``` + +Then run: + +```bash +freqtrade download-data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. +- To change the exchange used to download the tickers, please use a different configuration file (you'll probably need to adjust ratelimits etc.) +- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10` (defaults to 30 days). +- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. +- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. + ## Test your strategy with Backtesting -Now you have good Buy and Sell strategies, you want to test it against +Now you have good Buy and Sell strategies and some historic data, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). @@ -109,37 +143,6 @@ The full timerange specification: - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` -#### Downloading new set of ticker data - -To download new set of backtesting ticker data, you can use a download script. - -If you are using Binance for example: - -- create a directory `user_data/data/binance` and copy `pairs.json` in that directory. -- update the `pairs.json` to contain the currency pairs you are interested in. - -```bash -mkdir -p user_data/data/binance -cp freqtrade/tests/testdata/pairs.json user_data/data/binance -``` - -Then run: - -```bash -python scripts/download_backtest_data.py --exchange binance -``` - -This will download ticker data for all the currency pairs you defined in `pairs.json`. - -- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. -- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. -- To download ticker data for only 10 days, use `--days 10`. -- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. -- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with other options. - -For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). - ## Understand the backtesting result The most important in the backtesting is to understand the result. diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index ed12b6501..f7c393b60 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -281,6 +281,9 @@ class Configuration(object): self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='erase', + logstring='Erase detected. Deleting existing data.') + self._args_to_config(config, argname='timeframes', logstring='timeframes --timeframes: {}') diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 899c6d0c8..c7b3a28b0 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -122,7 +122,7 @@ def load_pair_history(pair: str, else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data' ) return None diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 4ba65e470..ea56b4bec 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -74,7 +74,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: assert ld is None assert log_has( 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data', caplog ) @@ -109,7 +109,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau assert os.path.isfile(file) is False assert log_has( 'No history data for pair: "MEME/BTC", interval: 1m. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data', caplog ) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d2770ba1a..7ccbae81a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -48,7 +48,7 @@ def start_list_exchanges(args: Namespace) -> None: def start_download_data(args: Namespace) -> None: """ - Download data based + Download data (former download_backtest_data.py script) """ config = setup_utils_configuration(args, RunMode.OTHER) @@ -76,7 +76,7 @@ def start_download_data(args: Namespace) -> None: pair_print = pair.replace('/', '_') filename = f'{pair_print}-{ticker_interval}.json' dl_file = dl_path.joinpath(filename) - if args.erase and dl_file.exists(): + if config.get("erase") and dl_file.exists(): logger.info( f'Deleting existing data for pair {pair}, interval {ticker_interval}.') dl_file.unlink() From 89257832d721cb06c7e2f80f860f8006fc856fc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:27:59 +0200 Subject: [PATCH 07/17] Don't use internal _API methods --- freqtrade/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 7ccbae81a..002d89738 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -68,7 +68,7 @@ def start_download_data(args: Namespace) -> None: exchange = ExchangeResolver(config['exchange']['name'], config).exchange for pair in config["pairs"]: - if pair not in exchange._api.markets: + if pair not in exchange.markets: pairs_not_available.append(pair) logger.info(f"Skipping pair {pair}...") continue From b2c215029d4937223b88b39a7d37c2373896be0d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:28:11 +0200 Subject: [PATCH 08/17] Add tests for download_data entrypoint --- freqtrade/tests/test_utils.py | 51 ++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index a12b709d7..8d0e76cde 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -1,8 +1,11 @@ -from freqtrade.utils import setup_utils_configuration, start_list_exchanges -from freqtrade.tests.conftest import get_args -from freqtrade.state import RunMode - import re +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +from freqtrade.state import RunMode +from freqtrade.tests.conftest import get_args, log_has, patch_exchange +from freqtrade.utils import (setup_utils_configuration, start_download_data, + start_list_exchanges) def test_setup_utils_configuration(): @@ -40,3 +43,43 @@ def test_list_exchanges(capsys): assert not re.match(r"Exchanges supported by ccxt and available.*", captured.out) assert re.search(r"^binance$", captured.out, re.MULTILINE) assert re.search(r"^bittrex$", captured.out, re.MULTILINE) + + +def test_download_data(mocker, markets, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--erase" + ] + start_download_data(get_args(args)) + + assert dl_mock.call_count == 4 + assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_no_markets(mocker, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC" + ] + start_download_data(get_args(args)) + + assert dl_mock.call_count == 0 + assert log_has("Skipping pair ETH/BTC...", caplog) + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) From 132f28ad44be07194ee68cda69a488c75d1b7b8b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:52:59 +0200 Subject: [PATCH 09/17] Add tests to correctly load / override pair-lists --- freqtrade/configuration/configuration.py | 21 ++++-- freqtrade/tests/test_configuration.py | 88 ++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f7c393b60..cb698544d 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -4,6 +4,7 @@ This module contains the configuration class import logging import warnings from argparse import Namespace +from pathlib import Path from typing import Any, Callable, Dict, List, Optional from freqtrade import OperationalException, constants @@ -12,7 +13,7 @@ from freqtrade.configuration.create_datadir import create_datadir from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.configuration.load_config import load_config_file from freqtrade.loggers import setup_logging -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, json_load from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -363,22 +364,28 @@ class Configuration(object): * whitelist from config """ - if "pairs" in self.args and self.args.pairs: + if "pairs" in config: return if "pairs_file" in self.args and self.args.pairs_file: - pairs_file = self.args.pairs_file + pairs_file = Path(self.args.pairs_file) logger.info(f'Reading pairs file "{pairs_file}".') # Download pairs from the pairs file if no config is specified # or if pairs file is specified explicitely if not pairs_file.exists(): - OperationalException(f'No pairs file found with path "{pairs_file}".') + raise OperationalException(f'No pairs file found with path "{pairs_file}".') - # with pairs_file.open() as file: - # pairs = list(set(json.load(file))) + config['pairs'] = json_load(pairs_file) - # pairs.sort() + config['pairs'].sort() + return if "config" in self.args: logger.info("Using pairlist from configuration.") config['pairs'] = config.get('exchange', {}).get('pair_whitelist') + else: + # Fall back to /dl_path/pairs.json + pairs_file = Path(config['datadir']) / "pairs.json" + if pairs_file.exists(): + config['pairs'] = json_load(pairs_file) + diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 8cbd02ece..c351b9b72 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -704,3 +704,91 @@ def test_load_config_default_subkeys(all_conf, keys) -> None: validate_config_schema(all_conf) assert subkey in all_conf[key] assert all_conf[key][subkey] == keys[2] + + +def test_pairlist_resolving(): + arglist = [ + 'download-data', + '--pairs', 'ETH/BTC', 'XRP/BTC', + '--exchange', 'binance' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == 'binance' + + +def test_pairlist_resolving_with_config(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + arglist = [ + '--config', 'config.json', + 'download-data', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == default_conf['exchange']['pair_whitelist'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + # Override pairs + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs', 'ETH/BTC', 'XRP/BTC', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + +def test_pairlist_resolving_with_config_pl(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + load_mock = mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs-file', 'pairs.json', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert load_mock.call_count == 1 + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + +def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs-file', 'pairs.json', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + with pytest.raises(OperationalException, match=r"No pairs file found with path.*"): + configuration = Configuration(args) + configuration.get_config() From c9207bcc0070bd574bc301f984955bbf3cdda0cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 16:01:30 +0200 Subject: [PATCH 10/17] Remove blank line at end --- freqtrade/configuration/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index cb698544d..c51153f4b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -388,4 +388,3 @@ class Configuration(object): pairs_file = Path(config['datadir']) / "pairs.json" if pairs_file.exists(): config['pairs'] = json_load(pairs_file) - From 29c56f4447ad53cb1d9dc157227be4ea09b4ab29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 06:47:53 +0200 Subject: [PATCH 11/17] Replace download_backtest_data script with warning message --- scripts/download_backtest_data.py | 145 ++---------------------------- 1 file changed, 5 insertions(+), 140 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index f77ad7422..496f83c7d 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,144 +1,9 @@ -#!/usr/bin/env python3 -""" -This script generates json files with pairs history data -""" -import arrow -import json import sys -from pathlib import Path -from typing import Any, Dict, List -from freqtrade.configuration import Arguments, TimeRange -from freqtrade.configuration import Configuration -from freqtrade.configuration.arguments import ARGS_DOWNLOADER -from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.load_config import load_config_file -from freqtrade.data.history import download_pair_history -from freqtrade.exchange import Exchange -from freqtrade.misc import deep_merge_dicts -import logging +print("This script has been integrated into freqtrade " + "and it's functionality is available by calling `freqtrade download-data`.") +print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ " + "for details.") -logger = logging.getLogger('download_backtest_data') - -DEFAULT_DL_PATH = 'user_data/data' - -# Do not read the default config if config is not specified -# in the command line options explicitely -arguments = Arguments(sys.argv[1:], 'Download backtest data', - no_default_config=True) -arguments._build_args(optionlist=ARGS_DOWNLOADER) -args = arguments._parse_args() - -# Use bittrex as default exchange -exchange_name = args.exchange or 'bittrex' - -pairs: List = [] - -configuration = Configuration(args) -config: Dict[str, Any] = {} - -if args.config: - # Now expecting a list of config filenames here, not a string - for path in args.config: - logger.info(f"Using config: {path}...") - # Merge config options, overwriting old values - config = deep_merge_dicts(load_config_file(path), config) - - config['stake_currency'] = '' - # Ensure we do not use Exchange credentials - config['exchange']['dry_run'] = True - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - - pairs = config['exchange']['pair_whitelist'] - - if config.get('ticker_interval'): - timeframes = args.timeframes or [config.get('ticker_interval')] - else: - timeframes = args.timeframes or ['1m', '5m'] - -else: - config = { - 'stake_currency': '', - 'dry_run': True, - 'exchange': { - 'name': exchange_name, - 'key': '', - 'secret': '', - 'pair_whitelist': [], - 'ccxt_async_config': { - 'enableRateLimit': True, - 'rateLimit': 200 - } - } - } - timeframes = args.timeframes or ['1m', '5m'] - -configuration._process_logging_options(config) - -if args.config and args.exchange: - logger.warning("The --exchange option is ignored, " - "using exchange settings from the configuration file.") - -# Check if the exchange set by the user is supported -check_exchange(config) - -configuration._process_datadir_options(config) - -dl_path = Path(config['datadir']) - -pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') - -if not pairs or args.pairs_file: - logger.info(f'Reading pairs file "{pairs_file}".') - # Download pairs from the pairs file if no config is specified - # or if pairs file is specified explicitely - if not pairs_file.exists(): - sys.exit(f'No pairs file found with path "{pairs_file}".') - - with pairs_file.open() as file: - pairs = list(set(json.load(file))) - - pairs.sort() - -timerange = TimeRange() -if args.days: - time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - -logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') - -pairs_not_available = [] - -try: - # Init exchange - exchange = Exchange(config) - - for pair in pairs: - if pair not in exchange._api.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for ticker_interval in timeframes: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if args.erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {ticker_interval}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=str(ticker_interval), - timerange=timerange) - -except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - -finally: - if pairs_not_available: - logger.info( - f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") +sys.exit(1) From f7d5280f47ced954a57536cde7dc1cc6561f7203 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 06:48:34 +0200 Subject: [PATCH 12/17] Replace ARGS_DOWNLOADER with ARGS_DOWNLOAD_DATA --- freqtrade/configuration/arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 8fa16318a..c45e3d7ba 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,7 +30,7 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_EXCHANGES = ["print_one_column"] -ARGS_DOWNLOADER = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", @@ -131,4 +131,4 @@ class Arguments(object): help='Download backtesting data.' ) download_data_cmd.set_defaults(func=start_download_data) - self._build_args(optionlist=ARGS_DOWNLOADER, parser=download_data_cmd) + self._build_args(optionlist=ARGS_DOWNLOAD_DATA, parser=download_data_cmd) From a53e9e3a98cf440d1f14a9d55d907a4139e46b97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 06:58:38 +0200 Subject: [PATCH 13/17] improve tests for download_module --- freqtrade/tests/test_utils.py | 50 +++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index 8d0e76cde..003b3b286 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -2,6 +2,8 @@ import re from pathlib import Path from unittest.mock import MagicMock, PropertyMock +import pytest + from freqtrade.state import RunMode from freqtrade.tests.conftest import get_args, log_has, patch_exchange from freqtrade.utils import (setup_utils_configuration, start_download_data, @@ -58,15 +60,41 @@ def test_download_data(mocker, markets, caplog): "download-data", "--exchange", "binance", "--pairs", "ETH/BTC", "XRP/BTC", - "--erase" + "--erase", ] start_download_data(get_args(args)) assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype is None + assert dl_mock.call_args[1]['timerange'].stoptype is None assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) +def test_download_data_days(mocker, markets, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + + start_download_data(get_args(args)) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype == 'date' + + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + def test_download_data_no_markets(mocker, caplog): dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) patch_exchange(mocker) @@ -76,10 +104,28 @@ def test_download_data_no_markets(mocker, caplog): args = [ "download-data", "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC" + "--pairs", "ETH/BTC", "XRP/BTC", ] start_download_data(get_args(args)) assert dl_mock.call_count == 0 assert log_has("Skipping pair ETH/BTC...", caplog) assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) + + +def test_download_data_keyboardInterrupt(mocker, caplog, markets): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', + MagicMock(side_effect=KeyboardInterrupt)) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + ] + with pytest.raises(SystemExit): + start_download_data(get_args(args)) + + assert dl_mock.call_count == 1 From 7a79b292e46dfab4e19e4635c8720bac8cde7676 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 07:05:42 +0200 Subject: [PATCH 14/17] Fix bug in pairs fallback resolving --- freqtrade/configuration/configuration.py | 3 ++- freqtrade/tests/test_configuration.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c51153f4b..676b0c594 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -380,7 +380,7 @@ class Configuration(object): config['pairs'].sort() return - if "config" in self.args: + if "config" in self.args and self.args.config: logger.info("Using pairlist from configuration.") config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: @@ -388,3 +388,4 @@ class Configuration(object): pairs_file = Path(config['datadir']) / "pairs.json" if pairs_file.exists(): config['pairs'] = json_load(pairs_file) + config['pairs'].sort() diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index c351b9b72..b6e8a76d9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -792,3 +792,21 @@ def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf): with pytest.raises(OperationalException, match=r"No pairs file found with path.*"): configuration = Configuration(args) configuration.get_config() + + +def test_pairlist_resolving_fallback(mocker): + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + arglist = [ + 'download-data', + '--exchange', 'binance' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == 'binance' From 08fa5136e11843c7ea3da2c464f1a0e77d921f03 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 07:19:46 +0200 Subject: [PATCH 15/17] use copy of minimal_config ... --- freqtrade/configuration/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 676b0c594..75319ac47 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -54,7 +54,7 @@ class Configuration(object): config: Dict[str, Any] = {} if not files: - return constants.MINIMAL_CONFIG + return constants.MINIMAL_CONFIG.copy() # We expect here a list of config filenames for path in files: From 5e440a4cdc6baa4ecadcc37593f373fe06aeb257 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 06:55:19 +0200 Subject: [PATCH 16/17] Improve docs to point to `freqtrade download-data` --- docs/backtesting.md | 2 +- docs/bot-usage.md | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index f666c5b49..543422fee 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -3,7 +3,7 @@ This page explains how to validate your strategy performance by using Backtesting. -## Getting data for backtesting / hyperopt +## Getting data for backtesting and hyperopt To download backtesting data (candles / OHLCV), we recommend using the `freqtrade download-data` command. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 31c5812ad..2873f5e8f 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -184,19 +184,11 @@ optional arguments: result.json) ``` -### How to use **--refresh-pairs-cached** parameter? +### Getting historic data for backtesting -The first time your run Backtesting, it will take the pairs you have -set in your config file and download data from the Exchange. - -If for any reason you want to update your data set, you use -`--refresh-pairs-cached` to force Backtesting to update the data it has. - -!!! Note - Use it only if you want to update your data set. You will not be able to come back to the previous version. - -To test your strategy with latest data, we recommend continuing using -the parameter `-l` or `--live`. +The first time your run Backtesting, you will need to download some historic data first. +This can be accomplished by using `freqtrade download-data`. +Check the corresponding [help page section](backtesting.md#Getting-data-for-backtesting-and-hyperopt) for more details ## Hyperopt commands From 8d1a575a9b975e1753d312fbf72d8e4f640354f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Aug 2019 06:39:28 +0200 Subject: [PATCH 17/17] Reword documentation --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 543422fee..3712fddce 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -5,7 +5,7 @@ Backtesting. ## Getting data for backtesting and hyperopt -To download backtesting data (candles / OHLCV), we recommend using the `freqtrade download-data` command. +To download backtesting data (candles / OHLCV) and hyperoptimization using the `freqtrade download-data` command. If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory.