Merge pull request #4780 from rokups/rk/new-pairs-days
Add --new-pairs-days parameter for download-data command.
This commit is contained in:
commit
f5f0bf53af
@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory.
|
||||
You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used.
|
||||
|
||||
!!! Tip "Tip: Updating existing data"
|
||||
If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
|
||||
Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
|
||||
If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data.
|
||||
If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only.
|
||||
If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data.
|
||||
|
||||
### Usage
|
||||
|
||||
@ -20,8 +21,9 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`-
|
||||
usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
|
||||
[-d PATH] [--userdir PATH]
|
||||
[-p PAIRS [PAIRS ...]] [--pairs-file FILE]
|
||||
[--days INT] [--timerange TIMERANGE]
|
||||
[--dl-trades] [--exchange EXCHANGE]
|
||||
[--days INT] [--new-pairs-days INT]
|
||||
[--timerange TIMERANGE] [--dl-trades]
|
||||
[--exchange EXCHANGE]
|
||||
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
|
||||
[--erase]
|
||||
[--data-format-ohlcv {json,jsongz,hdf5}]
|
||||
@ -34,6 +36,8 @@ optional arguments:
|
||||
separated.
|
||||
--pairs-file FILE File containing a list of pairs to download.
|
||||
--days INT Download data for given number of days.
|
||||
--new-pairs-days INT Download data of new pairs for given number of days.
|
||||
Default: `None`.
|
||||
--timerange TIMERANGE
|
||||
Specify what timerange of data to use.
|
||||
--dl-trades Download trades instead of OHLCV data. The bot will
|
||||
|
@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
|
||||
|
||||
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
|
||||
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
|
||||
"timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
|
||||
"download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
|
||||
"dataformat_trades"]
|
||||
|
||||
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
|
||||
"db_url", "trade_source", "export", "exportfilename",
|
||||
|
@ -345,6 +345,12 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
type=check_int_positive,
|
||||
metavar='INT',
|
||||
),
|
||||
"new_pairs_days": Arg(
|
||||
'--new-pairs-days',
|
||||
help='Download data of new pairs for given number of days. Default: `%(default)s`.',
|
||||
type=check_int_positive,
|
||||
metavar='INT',
|
||||
),
|
||||
"download_trades": Arg(
|
||||
'--dl-trades',
|
||||
help='Download trades instead of OHLCV data. The bot will resample trades to the '
|
||||
|
@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
if config.get('download_trades'):
|
||||
pairs_not_available = refresh_backtest_trades_data(
|
||||
exchange, pairs=expanded_pairs, datadir=config['datadir'],
|
||||
timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format=config['dataformat_trades'])
|
||||
timerange=timerange, new_pairs_days=config['new_pairs_days'],
|
||||
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
|
||||
|
||||
# Convert downloaded trade data to different timeframes
|
||||
convert_trades_to_ohlcv(
|
||||
@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
else:
|
||||
pairs_not_available = refresh_backtest_ohlcv_data(
|
||||
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
|
||||
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format=config['dataformat_ohlcv'])
|
||||
datadir=config['datadir'], timerange=timerange,
|
||||
new_pairs_days=config['new_pairs_days'],
|
||||
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
|
||||
|
||||
except KeyboardInterrupt:
|
||||
sys.exit("SIGINT received, aborting ...")
|
||||
|
@ -108,6 +108,8 @@ class Configuration:
|
||||
|
||||
self._process_plot_options(config)
|
||||
|
||||
self._process_data_options(config)
|
||||
|
||||
# Check if the exchange set by the user is supported
|
||||
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
|
||||
|
||||
@ -399,6 +401,11 @@ class Configuration:
|
||||
self._args_to_config(config, argname='dataformat_trades',
|
||||
logstring='Using "{}" to store trades data.')
|
||||
|
||||
def _process_data_options(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
self._args_to_config(config, argname='new_pairs_days',
|
||||
logstring='Detected --new-pairs-days: {}')
|
||||
|
||||
def _process_runmode(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
self._args_to_config(config, argname='dry_run',
|
||||
|
@ -96,6 +96,7 @@ CONF_SCHEMA = {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1},
|
||||
'new_pairs_days': {'type': 'integer', 'default': 30},
|
||||
'timeframe': {'type': 'string'},
|
||||
'stake_currency': {'type': 'string'},
|
||||
'stake_amount': {
|
||||
|
@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
|
||||
def _download_pair_history(datadir: Path,
|
||||
exchange: Exchange,
|
||||
pair: str, *,
|
||||
new_pairs_days: int = 30,
|
||||
timeframe: str = '5m',
|
||||
timerange: Optional[TimeRange] = None,
|
||||
data_handler: IDataHandler = None) -> bool:
|
||||
@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path,
|
||||
timeframe=timeframe,
|
||||
since_ms=since_ms if since_ms else
|
||||
int(arrow.utcnow().shift(
|
||||
days=-30).float_timestamp) * 1000
|
||||
days=-new_pairs_days).float_timestamp) * 1000
|
||||
)
|
||||
# TODO: Maybe move parsing to exchange class (?)
|
||||
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
|
||||
@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path,
|
||||
|
||||
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
|
||||
datadir: Path, timerange: Optional[TimeRange] = None,
|
||||
erase: bool = False, data_format: str = None) -> List[str]:
|
||||
new_pairs_days: int = 30, erase: bool = False,
|
||||
data_format: str = None) -> List[str]:
|
||||
"""
|
||||
Refresh stored ohlcv data for backtesting and hyperopt operations.
|
||||
Used by freqtrade download-data subcommand.
|
||||
@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
|
||||
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
|
||||
_download_pair_history(datadir=datadir, exchange=exchange,
|
||||
pair=pair, timeframe=str(timeframe),
|
||||
new_pairs_days=new_pairs_days,
|
||||
timerange=timerange, data_handler=data_handler)
|
||||
return pairs_not_available
|
||||
|
||||
|
||||
def _download_trades_history(exchange: Exchange,
|
||||
pair: str, *,
|
||||
new_pairs_days: int = 30,
|
||||
timerange: Optional[TimeRange] = None,
|
||||
data_handler: IDataHandler
|
||||
) -> bool:
|
||||
@ -263,7 +267,7 @@ def _download_trades_history(exchange: Exchange,
|
||||
|
||||
since = timerange.startts * 1000 if \
|
||||
(timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
|
||||
days=-30).float_timestamp) * 1000
|
||||
days=-new_pairs_days).float_timestamp) * 1000
|
||||
|
||||
trades = data_handler.trades_load(pair)
|
||||
|
||||
@ -311,8 +315,8 @@ def _download_trades_history(exchange: Exchange,
|
||||
|
||||
|
||||
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
|
||||
timerange: TimeRange, erase: bool = False,
|
||||
data_format: str = 'jsongz') -> List[str]:
|
||||
timerange: TimeRange, new_pairs_days: int = 30,
|
||||
erase: bool = False, data_format: str = 'jsongz') -> List[str]:
|
||||
"""
|
||||
Refresh stored trades data for backtesting and hyperopt operations.
|
||||
Used by freqtrade download-data subcommand.
|
||||
@ -333,6 +337,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
|
||||
logger.info(f'Downloading trades for pair {pair}.')
|
||||
_download_trades_history(exchange=exchange,
|
||||
pair=pair,
|
||||
new_pairs_days=new_pairs_days,
|
||||
timerange=timerange,
|
||||
data_handler=data_handler)
|
||||
return pairs_not_available
|
||||
|
Loading…
Reference in New Issue
Block a user