Add prepend option to download-data

freqtrade/commands/arguments.py

@@ -72,7 +72,8 @@ ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"]
 
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive",
                       "timerange", "download_trades", "exchange", "timeframes",
-                      "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"]
+                      "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode",
+                      "prepend_data"]
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
                        "db_url", "trade_source", "export", "exportfilename",

freqtrade/commands/cli_options.py

@@ -443,6 +443,11 @@ AVAILABLE_CLI_OPTIONS = {
         default=['1m', '5m'],
         nargs='+',
     ),
+    "prepend_data": Arg(
+        '--prepend',
+        help='Allow data prepending.',
+        action='store_true',
+    ),
     "erase": Arg(
         '--erase',
         help='Clean all existing data for the selected exchange/pairs/timeframes.',

freqtrade/commands/data_commands.py

@@ -85,6 +85,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
                 new_pairs_days=config['new_pairs_days'],
                 erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
                 trading_mode=config.get('trading_mode', 'spot'),
+                prepend=config.get('prepend_data', False)
             )
 
     except KeyboardInterrupt:

freqtrade/configuration/configuration.py

@@ -393,6 +393,8 @@ class Configuration:
         self._args_to_config(config, argname='trade_source',
                              logstring='Using trades from: {}')
 
+        self._args_to_config(config, argname='prepend_data',
+                             logstring='Prepend detected. Allowing data prepending.')
         self._args_to_config(config, argname='erase',
                              logstring='Erase detected. Deleting existing data.')
 

freqtrade/data/history/history_utils.py

@@ -172,7 +172,6 @@ def _load_cached_data_for_updating(
                 end = data.iloc[0]['date']
             else:
                 start = data.iloc[-1]['date']
-
     start_ms = int(start.timestamp() * 1000) if start else None
     end_ms = int(end.timestamp() * 1000) if end else None
     return data, start_ms, end_ms
@@ -188,6 +187,7 @@ def _download_pair_history(pair: str, *,
                            timerange: Optional[TimeRange] = None,
                            candle_type: CandleType,
                            erase: bool = False,
+                           prepend: bool = False,
                            ) -> bool:
     """
     Download latest candles from the exchange for the pair and timeframe passed in parameters
@@ -195,8 +195,6 @@ def _download_pair_history(pair: str, *,
     exists in a cache. If timerange starts earlier than the data in the cache,
     the full data will be redownloaded
 
-    Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
-
     :param pair: pair to download
     :param timeframe: Timeframe (e.g "5m")
     :param timerange: range of time to download
@@ -211,17 +209,17 @@ def _download_pair_history(pair: str, *,
             if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type):
                 logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.')
 
-        logger.info(
-            f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, '
-            f'candle type: {candle_type} and store in {datadir}.'
-        )
-
         data, since_ms, until_ms = _load_cached_data_for_updating(
             pair, timeframe, timerange,
             data_handler=data_handler,
             candle_type=candle_type,
-            prepend=False)
-        # TODO: Prepend should come from a param
+            prepend=prepend)
+
+        logger.info(f'Download history data for "{pair}" ({process}), timeframe: {timeframe}, '
+                    f'candle type: {candle_type} and store in {datadir}.'
+                    f'From {format_ms_time(since_ms) if since_ms else "start"} to '
+                    f'{format_ms_time(until_ms) if until_ms else "now"}'
+                    )
 
         logger.debug("Current Start: %s",
                      f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
@@ -269,6 +267,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
                                 timerange: Optional[TimeRange] = None,
                                 new_pairs_days: int = 30, erase: bool = False,
                                 data_format: str = None,
+                                prepend: bool = False,
                                 ) -> List[str]:
     """
     Refresh stored ohlcv data for backtesting and hyperopt operations.
@@ -292,7 +291,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
                                    timerange=timerange, data_handler=data_handler,
                                    timeframe=str(timeframe), new_pairs_days=new_pairs_days,
                                    candle_type=candle_type,
-                                   erase=erase)
+                                   erase=erase, prepend=prepend)
         if trading_mode == 'futures':
             # Predefined candletype (and timeframe) depending on exchange
             # Downloads what is necessary to backtest based on futures data.
@@ -306,7 +305,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
                                        timerange=timerange, data_handler=data_handler,
                                        timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
                                        candle_type=funding_candle_type,
-                                       erase=erase)
+                                       erase=erase, prepend=prepend)
 
     return pairs_not_available