Merge pull request #4780 from rokups/rk/new-pairs-days

Add --new-pairs-days parameter for download-data command.

docs/data-download.md

@@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory.
 You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used.
 
 !!! Tip "Tip: Updating existing data"
-    If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
+    If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data.
-    Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
+    If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only.
+    If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data.
 
 ### Usage
 
@@ -20,8 +21,9 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`-
 usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [-d PATH] [--userdir PATH]
                                [-p PAIRS [PAIRS ...]] [--pairs-file FILE]
-                               [--days INT] [--timerange TIMERANGE]
+                               [--days INT] [--new-pairs-days INT]
-                               [--dl-trades] [--exchange EXCHANGE]
+                               [--timerange TIMERANGE] [--dl-trades]
+                               [--exchange EXCHANGE]
                                [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
                                [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5}]
@@ -34,6 +36,8 @@ optional arguments:
                         separated.
   --pairs-file FILE     File containing a list of pairs to download.
   --days INT            Download data for given number of days.
+  --new-pairs-days INT  Download data of new pairs for given number of days.
+                        Default: `None`.
   --timerange TIMERANGE
                         Specify what timerange of data to use.
   --dl-trades           Download trades instead of OHLCV data. The bot will

freqtrade/commands/arguments.py

@@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
 
-ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
+ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
-                      "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
+                      "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
+                      "dataformat_trades"]
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
                        "db_url", "trade_source", "export", "exportfilename",

freqtrade/commands/cli_options.py

@@ -345,6 +345,12 @@ AVAILABLE_CLI_OPTIONS = {
         type=check_int_positive,
         metavar='INT',
     ),
+    "new_pairs_days": Arg(
+        '--new-pairs-days',
+        help='Download data of new pairs for given number of days. Default: `%(default)s`.',
+        type=check_int_positive,
+        metavar='INT',
+    ),
     "download_trades": Arg(
         '--dl-trades',
         help='Download trades instead of OHLCV data. The bot will resample trades to the '

freqtrade/commands/data_commands.py

@@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
         if config.get('download_trades'):
             pairs_not_available = refresh_backtest_trades_data(
                 exchange, pairs=expanded_pairs, datadir=config['datadir'],
-                timerange=timerange, erase=bool(config.get('erase')),
+                timerange=timerange, new_pairs_days=config['new_pairs_days'],
-                data_format=config['dataformat_trades'])
+                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
@@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
         else:
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
+                datadir=config['datadir'], timerange=timerange,
-                data_format=config['dataformat_ohlcv'])
+                new_pairs_days=config['new_pairs_days'],
+                erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
 
     except KeyboardInterrupt:
         sys.exit("SIGINT received, aborting ...")

freqtrade/configuration/configuration.py

@@ -108,6 +108,8 @@ class Configuration:
 
         self._process_plot_options(config)
 
+        self._process_data_options(config)
+
         # Check if the exchange set by the user is supported
         check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
 
@@ -399,6 +401,11 @@ class Configuration:
         self._args_to_config(config, argname='dataformat_trades',
                              logstring='Using "{}" to store trades data.')
 
+    def _process_data_options(self, config: Dict[str, Any]) -> None:
+
+        self._args_to_config(config, argname='new_pairs_days',
+                             logstring='Detected --new-pairs-days: {}')
+
     def _process_runmode(self, config: Dict[str, Any]) -> None:
 
         self._args_to_config(config, argname='dry_run',

freqtrade/constants.py

@@ -96,6 +96,7 @@ CONF_SCHEMA = {
     'type': 'object',
     'properties': {
         'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1},
+        'new_pairs_days': {'type': 'integer', 'default': 30},
         'timeframe': {'type': 'string'},
         'stake_currency': {'type': 'string'},
         'stake_amount': {

freqtrade/data/history/history_utils.py

@@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
 def _download_pair_history(datadir: Path,
                            exchange: Exchange,
                            pair: str, *,
+                           new_pairs_days: int = 30,
                            timeframe: str = '5m',
                            timerange: Optional[TimeRange] = None,
                            data_handler: IDataHandler = None) -> bool:
@@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path,
                                                timeframe=timeframe,
                                                since_ms=since_ms if since_ms else
                                                int(arrow.utcnow().shift(
-                                                   days=-30).float_timestamp) * 1000
+                                                   days=-new_pairs_days).float_timestamp) * 1000
                                                )
         # TODO: Maybe move parsing to exchange class (?)
         new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path,
 
 def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
                                 datadir: Path, timerange: Optional[TimeRange] = None,
-                                erase: bool = False, data_format: str = None) -> List[str]:
+                                new_pairs_days: int = 30, erase: bool = False,
+                                data_format: str = None) -> List[str]:
     """
     Refresh stored ohlcv data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
             logger.info(f'Downloading pair {pair}, interval {timeframe}.')
             _download_pair_history(datadir=datadir, exchange=exchange,
                                    pair=pair, timeframe=str(timeframe),
+                                   new_pairs_days=new_pairs_days,
                                    timerange=timerange, data_handler=data_handler)
     return pairs_not_available
 
 
 def _download_trades_history(exchange: Exchange,
                              pair: str, *,
+                             new_pairs_days: int = 30,
                              timerange: Optional[TimeRange] = None,
                              data_handler: IDataHandler
                              ) -> bool:
@@ -263,7 +267,7 @@ def _download_trades_history(exchange: Exchange,
 
         since = timerange.startts * 1000 if \
             (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
-                days=-30).float_timestamp) * 1000
+                days=-new_pairs_days).float_timestamp) * 1000
 
         trades = data_handler.trades_load(pair)
 
@@ -311,8 +315,8 @@ def _download_trades_history(exchange: Exchange,
 
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
-                                 timerange: TimeRange, erase: bool = False,
+                                 timerange: TimeRange, new_pairs_days: int = 30,
-                                 data_format: str = 'jsongz') -> List[str]:
+                                 erase: bool = False, data_format: str = 'jsongz') -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -333,6 +337,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
         logger.info(f'Downloading trades for pair {pair}.')
         _download_trades_history(exchange=exchange,
                                  pair=pair,
+                                 new_pairs_days=new_pairs_days,
                                  timerange=timerange,
                                  data_handler=data_handler)
     return pairs_not_available