Add --new-pairs-days parameter for download-data command.
This parameter allows us to customize a number of days we would like to download for new pairs only. This allows us to achieve efficient data update, downloading all data for new pairs and only missing data for existing pairs. To do that use `freqtrade download-data --new-pairs-days=3650` (not specifying `--days` or `--timerange` causes freqtrade to download only missing data for existing pairs).
This commit is contained in:
parent
9f6f3e0862
commit
09efa7b06b
@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory.
|
||||
You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used.
|
||||
|
||||
!!! Tip "Tip: Updating existing data"
|
||||
If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
|
||||
Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
|
||||
If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data.
|
||||
If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only.
|
||||
If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data.
|
||||
|
||||
### Usage
|
||||
|
||||
@ -34,6 +35,7 @@ optional arguments:
|
||||
separated.
|
||||
--pairs-file FILE File containing a list of pairs to download.
|
||||
--days INT Download data for given number of days.
|
||||
--new-pairs-days INT Download data of new pairs for given number of days. Default: `30`.
|
||||
--timerange TIMERANGE
|
||||
Specify what timerange of data to use.
|
||||
--dl-trades Download trades instead of OHLCV data. The bot will
|
||||
|
@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
|
||||
|
||||
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
|
||||
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
|
||||
"timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
|
||||
"download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
|
||||
"dataformat_trades"]
|
||||
|
||||
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
|
||||
"db_url", "trade_source", "export", "exportfilename",
|
||||
|
@ -345,6 +345,13 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
type=check_int_positive,
|
||||
metavar='INT',
|
||||
),
|
||||
"new_pairs_days": Arg(
|
||||
'--new-pairs-days',
|
||||
help='Download data of new pairs for given number of days. Default: `%(default)s`.',
|
||||
type=check_int_positive,
|
||||
metavar='INT',
|
||||
default=30,
|
||||
),
|
||||
"download_trades": Arg(
|
||||
'--dl-trades',
|
||||
help='Download trades instead of OHLCV data. The bot will resample trades to the '
|
||||
|
@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
if config.get('download_trades'):
|
||||
pairs_not_available = refresh_backtest_trades_data(
|
||||
exchange, pairs=expanded_pairs, datadir=config['datadir'],
|
||||
timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format=config['dataformat_trades'])
|
||||
timerange=timerange, new_pairs_days=config['new_pairs_days'],
|
||||
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
|
||||
|
||||
# Convert downloaded trade data to different timeframes
|
||||
convert_trades_to_ohlcv(
|
||||
@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
else:
|
||||
pairs_not_available = refresh_backtest_ohlcv_data(
|
||||
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
|
||||
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format=config['dataformat_ohlcv'])
|
||||
datadir=config['datadir'], timerange=timerange,
|
||||
new_pairs_days=config['new_pairs_days'],
|
||||
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
|
||||
|
||||
except KeyboardInterrupt:
|
||||
sys.exit("SIGINT received, aborting ...")
|
||||
|
@ -108,6 +108,8 @@ class Configuration:
|
||||
|
||||
self._process_plot_options(config)
|
||||
|
||||
self._process_data_options(config)
|
||||
|
||||
# Check if the exchange set by the user is supported
|
||||
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
|
||||
|
||||
@ -399,6 +401,11 @@ class Configuration:
|
||||
self._args_to_config(config, argname='dataformat_trades',
|
||||
logstring='Using "{}" to store trades data.')
|
||||
|
||||
def _process_data_options(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
self._args_to_config(config, argname='new_pairs_days',
|
||||
logstring='Detected --new-pairs-days: {}')
|
||||
|
||||
def _process_runmode(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
self._args_to_config(config, argname='dry_run',
|
||||
|
@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
|
||||
def _download_pair_history(datadir: Path,
|
||||
exchange: Exchange,
|
||||
pair: str, *,
|
||||
new_pairs_days: int = 30,
|
||||
timeframe: str = '5m',
|
||||
timerange: Optional[TimeRange] = None,
|
||||
data_handler: IDataHandler = None) -> bool:
|
||||
@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path,
|
||||
timeframe=timeframe,
|
||||
since_ms=since_ms if since_ms else
|
||||
int(arrow.utcnow().shift(
|
||||
days=-30).float_timestamp) * 1000
|
||||
days=-new_pairs_days).float_timestamp) * 1000
|
||||
)
|
||||
# TODO: Maybe move parsing to exchange class (?)
|
||||
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
|
||||
@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path,
|
||||
|
||||
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
|
||||
datadir: Path, timerange: Optional[TimeRange] = None,
|
||||
erase: bool = False, data_format: str = None) -> List[str]:
|
||||
new_pairs_days: int = 30, erase: bool = False,
|
||||
data_format: str = None) -> List[str]:
|
||||
"""
|
||||
Refresh stored ohlcv data for backtesting and hyperopt operations.
|
||||
Used by freqtrade download-data subcommand.
|
||||
@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
|
||||
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
|
||||
_download_pair_history(datadir=datadir, exchange=exchange,
|
||||
pair=pair, timeframe=str(timeframe),
|
||||
new_pairs_days=new_pairs_days,
|
||||
timerange=timerange, data_handler=data_handler)
|
||||
return pairs_not_available
|
||||
|
||||
|
||||
def _download_trades_history(exchange: Exchange,
|
||||
pair: str, *,
|
||||
new_pairs_days: int = 30,
|
||||
timerange: Optional[TimeRange] = None,
|
||||
data_handler: IDataHandler
|
||||
) -> bool:
|
||||
@ -263,7 +267,7 @@ def _download_trades_history(exchange: Exchange,
|
||||
|
||||
since = timerange.startts * 1000 if \
|
||||
(timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
|
||||
days=-30).float_timestamp) * 1000
|
||||
days=-new_pairs_days).float_timestamp) * 1000
|
||||
|
||||
trades = data_handler.trades_load(pair)
|
||||
|
||||
@ -311,8 +315,8 @@ def _download_trades_history(exchange: Exchange,
|
||||
|
||||
|
||||
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
|
||||
timerange: TimeRange, erase: bool = False,
|
||||
data_format: str = 'jsongz') -> List[str]:
|
||||
timerange: TimeRange, new_pairs_days: int = 30,
|
||||
erase: bool = False, data_format: str = 'jsongz') -> List[str]:
|
||||
"""
|
||||
Refresh stored trades data for backtesting and hyperopt operations.
|
||||
Used by freqtrade download-data subcommand.
|
||||
@ -333,6 +337,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
|
||||
logger.info(f'Downloading trades for pair {pair}.')
|
||||
_download_trades_history(exchange=exchange,
|
||||
pair=pair,
|
||||
new_pairs_days=new_pairs_days,
|
||||
timerange=timerange,
|
||||
data_handler=data_handler)
|
||||
return pairs_not_available
|
||||
|
Loading…
Reference in New Issue
Block a user