From 09efa7b06b7cd39726d07c9f1f231249651271c8 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 22 Apr 2021 10:07:13 +0300 Subject: [PATCH 1/3] Add --new-pairs-days parameter for download-data command. This parameter allows us to customize a number of days we would like to download for new pairs only. This allows us to achieve efficient data update, downloading all data for new pairs and only missing data for existing pairs. To do that use `freqtrade download-data --new-pairs-days=3650` (not specifying `--days` or `--timerange` causes freqtrade to download only missing data for existing pairs). --- docs/data-download.md | 6 ++++-- freqtrade/commands/arguments.py | 5 +++-- freqtrade/commands/cli_options.py | 7 +++++++ freqtrade/commands/data_commands.py | 9 +++++---- freqtrade/configuration/configuration.py | 7 +++++++ freqtrade/data/history/history_utils.py | 15 ++++++++++----- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 7a78334d5..7b09cf49c 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory. You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used. !!! Tip "Tip: Updating existing data" - If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. - Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data. + If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only. + If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data. ### Usage @@ -34,6 +35,7 @@ optional arguments: separated. --pairs-file FILE File containing a list of pairs to download. --days INT Download data for given number of days. + --new-pairs-days INT Download data of new pairs for given number of days. Default: `30`. --timerange TIMERANGE Specify what timerange of data to use. --dl-trades Download trades instead of OHLCV data. The bot will diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9cf9992ce..ffd317799 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] -ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange", - "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange", + "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv", + "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index e49895de4..80c56ecfa 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -345,6 +345,13 @@ AVAILABLE_CLI_OPTIONS = { type=check_int_positive, metavar='INT', ), + "new_pairs_days": Arg( + '--new-pairs-days', + help='Download data of new pairs for given number of days. Default: `%(default)s`.', + type=check_int_positive, + metavar='INT', + default=30, + ), "download_trades": Arg( '--dl-trades', help='Download trades instead of OHLCV data. The bot will resample trades to the ' diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 1ce02eee5..58191ddb4 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], - timerange=timerange, erase=bool(config.get('erase')), - data_format=config['dataformat_trades']) + timerange=timerange, new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( @@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None: else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], - datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), - data_format=config['dataformat_ohlcv']) + datadir=config['datadir'], timerange=timerange, + new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index cc11f97c2..86f337c1b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -108,6 +108,8 @@ class Configuration: self._process_plot_options(config) + self._process_data_options(config) + # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -399,6 +401,11 @@ class Configuration: self._args_to_config(config, argname='dataformat_trades', logstring='Using "{}" to store trades data.') + def _process_data_options(self, config: Dict[str, Any]) -> None: + + self._args_to_config(config, argname='new_pairs_days', + logstring='Detected --new-pairs-days: {}') + def _process_runmode(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 3b8b5a2f0..58965abe0 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, *, + new_pairs_days: int = 30, timeframe: str = '5m', timerange: Optional[TimeRange] = None, data_handler: IDataHandler = None) -> bool: @@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path, timeframe=timeframe, since_ms=since_ms if since_ms else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000 + days=-new_pairs_days).float_timestamp) * 1000 ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, @@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], datadir: Path, timerange: Optional[TimeRange] = None, - erase: bool = False, data_format: str = None) -> List[str]: + new_pairs_days: int = 30, erase: bool = False, + data_format: str = None) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes logger.info(f'Downloading pair {pair}, interval {timeframe}.') _download_pair_history(datadir=datadir, exchange=exchange, pair=pair, timeframe=str(timeframe), + new_pairs_days=new_pairs_days, timerange=timerange, data_handler=data_handler) return pairs_not_available def _download_trades_history(exchange: Exchange, pair: str, *, + new_pairs_days: int = 30, timerange: Optional[TimeRange] = None, data_handler: IDataHandler ) -> bool: @@ -263,7 +267,7 @@ def _download_trades_history(exchange: Exchange, since = timerange.startts * 1000 if \ (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000 + days=-new_pairs_days).float_timestamp) * 1000 trades = data_handler.trades_load(pair) @@ -311,8 +315,8 @@ def _download_trades_history(exchange: Exchange, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, erase: bool = False, - data_format: str = 'jsongz') -> List[str]: + timerange: TimeRange, new_pairs_days: int = 30, + erase: bool = False, data_format: str = 'jsongz') -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -333,6 +337,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: logger.info(f'Downloading trades for pair {pair}.') _download_trades_history(exchange=exchange, pair=pair, + new_pairs_days=new_pairs_days, timerange=timerange, data_handler=data_handler) return pairs_not_available From 3144185409ddeb36529b693a7bcb19df400aac20 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 22 Apr 2021 11:18:28 +0300 Subject: [PATCH 2/3] Allow specifying "new_pairs_days" in config. --- freqtrade/commands/cli_options.py | 1 - freqtrade/constants.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 80c56ecfa..b583b47ba 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -350,7 +350,6 @@ AVAILABLE_CLI_OPTIONS = { help='Download data of new pairs for given number of days. Default: `%(default)s`.', type=check_int_positive, metavar='INT', - default=30, ), "download_trades": Arg( '--dl-trades', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index aea6e1ff2..77bd2a029 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -11,6 +11,7 @@ DEFAULT_EXCHANGE = 'bittrex' PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec +NEW_PAIRS_DAYS = 30 DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -96,6 +97,7 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, + 'new_pairs_days': {'type': 'integer', 'default': NEW_PAIRS_DAYS}, 'timeframe': {'type': 'string'}, 'stake_currency': {'type': 'string'}, 'stake_amount': { From ccaf5764da5f9d63c77bb901a0b89541cb3ea661 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 22 Apr 2021 19:41:01 +0200 Subject: [PATCH 3/3] Small adjustments --- docs/data-download.md | 8 +++++--- freqtrade/constants.py | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 7b09cf49c..01561c89b 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -21,8 +21,9 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`- usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] [--pairs-file FILE] - [--days INT] [--timerange TIMERANGE] - [--dl-trades] [--exchange EXCHANGE] + [--days INT] [--new-pairs-days INT] + [--timerange TIMERANGE] [--dl-trades] + [--exchange EXCHANGE] [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] [--erase] [--data-format-ohlcv {json,jsongz,hdf5}] @@ -35,7 +36,8 @@ optional arguments: separated. --pairs-file FILE File containing a list of pairs to download. --days INT Download data for given number of days. - --new-pairs-days INT Download data of new pairs for given number of days. Default: `30`. + --new-pairs-days INT Download data of new pairs for given number of days. + Default: `None`. --timerange TIMERANGE Specify what timerange of data to use. --dl-trades Download trades instead of OHLCV data. The bot will diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 77bd2a029..bfd1e72f1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -11,7 +11,6 @@ DEFAULT_EXCHANGE = 'bittrex' PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec -NEW_PAIRS_DAYS = 30 DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -97,7 +96,7 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, - 'new_pairs_days': {'type': 'integer', 'default': NEW_PAIRS_DAYS}, + 'new_pairs_days': {'type': 'integer', 'default': 30}, 'timeframe': {'type': 'string'}, 'stake_currency': {'type': 'string'}, 'stake_amount': {