diff --git a/docs/data-download.md b/docs/data-download.md index 9bfc1e685..c1caa8722 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -30,6 +30,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] [--trading-mode {spot,margin,futures}] + [--prepend] optional arguments: -h, --help show this help message and exit @@ -62,6 +63,7 @@ optional arguments: `jsongz`). --trading-mode {spot,margin,futures} Select Trading mode + --prepend Allow data prepending. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -157,10 +159,21 @@ freqtrade download-data --exchange binance --pairs .*/USDT - To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). -- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored. +- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +#### Download additional data before the current timerange + +Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data. +You can do so by using the `--prepend` flag, combined with + +``` bash +freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --prepend --timerange 20210101-20220101 +``` + +!!! Note + Freqtrade will ignore the end-date in this mode if data is available, updating the end-date to the existing data start point. ### Data format diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 62b79da2e..ff1d16590 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -72,7 +72,8 @@ ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", - "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"] + "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode", + "prepend_data"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index df8966e85..58e208652 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -443,6 +443,11 @@ AVAILABLE_CLI_OPTIONS = { default=['1m', '5m'], nargs='+', ), + "prepend_data": Arg( + '--prepend', + help='Allow data prepending.', + action='store_true', + ), "erase": Arg( '--erase', help='Clean all existing data for the selected exchange/pairs/timeframes.', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index e41512ccc..a2e2a100a 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -85,6 +85,7 @@ def start_download_data(args: Dict[str, Any]) -> None: new_pairs_days=config['new_pairs_days'], erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], trading_mode=config.get('trading_mode', 'spot'), + prepend=config.get('prepend_data', False) ) except KeyboardInterrupt: diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index dde56c220..80df6fb3f 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -393,6 +393,8 @@ class Configuration: self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='prepend_data', + logstring='Prepend detected. Allowing data prepending.') self._args_to_config(config, argname='erase', logstring='Erase detected. Deleting existing data.') diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index d4fe6322a..f1304607b 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -172,7 +172,6 @@ def _load_cached_data_for_updating( end = data.iloc[0]['date'] else: start = data.iloc[-1]['date'] - start_ms = int(start.timestamp() * 1000) if start else None end_ms = int(end.timestamp() * 1000) if end else None return data, start_ms, end_ms @@ -188,6 +187,7 @@ def _download_pair_history(pair: str, *, timerange: Optional[TimeRange] = None, candle_type: CandleType, erase: bool = False, + prepend: bool = False, ) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters @@ -195,8 +195,6 @@ def _download_pair_history(pair: str, *, exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded - Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download @@ -211,17 +209,17 @@ def _download_pair_history(pair: str, *, if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.') - logger.info( - f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, ' - f'candle type: {candle_type} and store in {datadir}.' - ) - data, since_ms, until_ms = _load_cached_data_for_updating( pair, timeframe, timerange, data_handler=data_handler, candle_type=candle_type, - prepend=False) - # TODO: Prepend should come from a param + prepend=prepend) + + logger.info(f'Download history data for "{pair}" ({process}), timeframe: {timeframe}, ' + f'candle type: {candle_type} and store in {datadir}.' + f'From {format_ms_time(since_ms) if since_ms else "start"} to ' + f'{format_ms_time(until_ms) if until_ms else "now"}' + ) logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') @@ -269,6 +267,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, data_format: str = None, + prepend: bool = False, ) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. @@ -292,7 +291,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange=timerange, data_handler=data_handler, timeframe=str(timeframe), new_pairs_days=new_pairs_days, candle_type=candle_type, - erase=erase) + erase=erase, prepend=prepend) if trading_mode == 'futures': # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. @@ -306,7 +305,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange=timerange, data_handler=data_handler, timeframe=str(tf_mark), new_pairs_days=new_pairs_days, candle_type=funding_candle_type, - erase=erase) + erase=erase, prepend=prepend) return pairs_not_available diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 689ffa4ce..31311cc38 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1983,6 +1983,20 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ assert exchange._api_async.fetch_ohlcv.call_count > 200 assert res[0] == ohlcv[0] + exchange._api_async.fetch_ohlcv.reset_mock() + end_ts = 1_500_500_000_000 + start_ts = 1_500_000_000_000 + respair, restf, _, res = await exchange._async_get_historic_ohlcv( + pair, "5m", since_ms=start_ts, candle_type=candle_type, is_new_pair=False, + until_ms=end_ts + ) + # Required candles + candles = (end_ts - start_ts) / 300_000 + exp = candles // exchange.ohlcv_candle_limit('5m') + 1 + + # Depending on the exchange, this should be called between 1 and 6 times. + assert exchange._api_async.fetch_ohlcv.call_count == exp + @pytest.mark.parametrize('candle_type', [CandleType.FUTURES, CandleType.MARK, CandleType.SPOT]) def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None: