diff --git a/docs/data-download.md b/docs/data-download.md index 7a78334d5..7b09cf49c 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory. You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used. !!! Tip "Tip: Updating existing data" - If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. - Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data. + If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only. + If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data. ### Usage @@ -34,6 +35,7 @@ optional arguments: separated. --pairs-file FILE File containing a list of pairs to download. --days INT Download data for given number of days. + --new-pairs-days INT Download data of new pairs for given number of days. Default: `30`. --timerange TIMERANGE Specify what timerange of data to use. --dl-trades Download trades instead of OHLCV data. The bot will diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9cf9992ce..ffd317799 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] -ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange", - "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange", + "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv", + "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index e49895de4..80c56ecfa 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -345,6 +345,13 @@ AVAILABLE_CLI_OPTIONS = { type=check_int_positive, metavar='INT', ), + "new_pairs_days": Arg( + '--new-pairs-days', + help='Download data of new pairs for given number of days. Default: `%(default)s`.', + type=check_int_positive, + metavar='INT', + default=30, + ), "download_trades": Arg( '--dl-trades', help='Download trades instead of OHLCV data. The bot will resample trades to the ' diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 1ce02eee5..58191ddb4 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], - timerange=timerange, erase=bool(config.get('erase')), - data_format=config['dataformat_trades']) + timerange=timerange, new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( @@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None: else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], - datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), - data_format=config['dataformat_ohlcv']) + datadir=config['datadir'], timerange=timerange, + new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index cc11f97c2..86f337c1b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -108,6 +108,8 @@ class Configuration: self._process_plot_options(config) + self._process_data_options(config) + # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -399,6 +401,11 @@ class Configuration: self._args_to_config(config, argname='dataformat_trades', logstring='Using "{}" to store trades data.') + def _process_data_options(self, config: Dict[str, Any]) -> None: + + self._args_to_config(config, argname='new_pairs_days', + logstring='Detected --new-pairs-days: {}') + def _process_runmode(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 3b8b5a2f0..58965abe0 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, *, + new_pairs_days: int = 30, timeframe: str = '5m', timerange: Optional[TimeRange] = None, data_handler: IDataHandler = None) -> bool: @@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path, timeframe=timeframe, since_ms=since_ms if since_ms else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000 + days=-new_pairs_days).float_timestamp) * 1000 ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, @@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], datadir: Path, timerange: Optional[TimeRange] = None, - erase: bool = False, data_format: str = None) -> List[str]: + new_pairs_days: int = 30, erase: bool = False, + data_format: str = None) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes logger.info(f'Downloading pair {pair}, interval {timeframe}.') _download_pair_history(datadir=datadir, exchange=exchange, pair=pair, timeframe=str(timeframe), + new_pairs_days=new_pairs_days, timerange=timerange, data_handler=data_handler) return pairs_not_available def _download_trades_history(exchange: Exchange, pair: str, *, + new_pairs_days: int = 30, timerange: Optional[TimeRange] = None, data_handler: IDataHandler ) -> bool: @@ -263,7 +267,7 @@ def _download_trades_history(exchange: Exchange, since = timerange.startts * 1000 if \ (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000 + days=-new_pairs_days).float_timestamp) * 1000 trades = data_handler.trades_load(pair) @@ -311,8 +315,8 @@ def _download_trades_history(exchange: Exchange, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, erase: bool = False, - data_format: str = 'jsongz') -> List[str]: + timerange: TimeRange, new_pairs_days: int = 30, + erase: bool = False, data_format: str = 'jsongz') -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -333,6 +337,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: logger.info(f'Downloading trades for pair {pair}.') _download_trades_history(exchange=exchange, pair=pair, + new_pairs_days=new_pairs_days, timerange=timerange, data_handler=data_handler) return pairs_not_available