stable/scripts/download_backtest_data.py

142 lines
4.3 KiB
Python
Raw Normal View History

2018-04-22 07:56:49 +00:00
#!/usr/bin/env python3
"""
2019-05-29 18:57:14 +00:00
This script generates json files with pairs history data
"""
2019-05-29 18:57:14 +00:00
import arrow
2018-04-22 07:56:49 +00:00
import json
import sys
2018-06-24 17:52:12 +00:00
from pathlib import Path
2019-05-29 18:57:14 +00:00
from typing import Any, Dict, List
2019-05-29 18:57:14 +00:00
from freqtrade.arguments import Arguments, TimeRange
from freqtrade.configuration import Configuration
from freqtrade.data.history import download_pair_history
2019-05-29 18:57:14 +00:00
from freqtrade.exchange import Exchange
from freqtrade.misc import deep_merge_dicts
2018-06-24 17:52:12 +00:00
import logging
2019-05-29 18:57:14 +00:00
logger = logging.getLogger('download_backtest_data')
2018-04-22 07:56:49 +00:00
DEFAULT_DL_PATH = 'user_data/data'
2018-04-22 07:56:49 +00:00
arguments = Arguments(sys.argv[1:], 'download utility')
2019-05-29 18:57:14 +00:00
arguments.download_data_options()
# Do not read the default config if config is not specified
# in the command line options explicitely
args = arguments.parse_args(no_default_config=True)
2018-04-22 07:56:49 +00:00
2019-06-11 07:09:30 +00:00
# Use bittrex as default exchange
exchange_name = args.exchange or 'bittrex'
2019-05-29 18:57:14 +00:00
pairs: List = []
2018-04-22 07:56:49 +00:00
2019-05-29 18:57:14 +00:00
configuration = Configuration(args)
config: Dict[str, Any] = {}
2019-05-29 18:57:14 +00:00
if args.config:
# Now expecting a list of config filenames here, not a string
for path in args.config:
2019-05-30 06:54:58 +00:00
logger.info(f"Using config: {path}...")
# Merge config options, overwriting old values
config = deep_merge_dicts(configuration._load_config_file(path), config)
config['stake_currency'] = ''
# Ensure we do not use Exchange credentials
2019-05-29 18:57:14 +00:00
config['exchange']['dry_run'] = True
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
2019-05-29 18:57:14 +00:00
pairs = config['exchange']['pair_whitelist']
2019-06-13 18:26:47 +00:00
2019-06-13 23:58:34 +00:00
if config.get('ticker_interval'):
timeframes = args.timeframes or [config.get('ticker_interval')]
else:
timeframes = args.timeframes or ['1m', '5m']
2019-05-29 18:57:14 +00:00
else:
config = {
'stake_currency': '',
'dry_run': True,
'exchange': {
2019-06-11 07:09:30 +00:00
'name': exchange_name,
'key': '',
'secret': '',
'pair_whitelist': [],
'ccxt_async_config': {
'enableRateLimit': True,
'rateLimit': 200
}
}
}
2019-06-13 23:58:34 +00:00
timeframes = args.timeframes or ['1m', '5m']
2019-06-13 18:26:47 +00:00
2019-05-29 18:57:14 +00:00
configuration._load_logging_config(config)
2019-06-11 07:09:30 +00:00
if args.config and args.exchange:
2019-06-11 07:21:59 +00:00
logger.warning("The --exchange option is ignored, "
"using exchange settings from the configuration file.")
2019-06-11 07:09:30 +00:00
# Check if the exchange set by the user is supported
configuration.check_exchange(config)
2019-05-29 18:57:14 +00:00
configuration._load_datadir_config(config)
2019-05-29 18:57:14 +00:00
dl_path = Path(config['datadir'])
2018-06-24 17:52:12 +00:00
pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
2018-04-22 07:56:49 +00:00
2019-05-29 18:57:14 +00:00
if not pairs or args.pairs_file:
2019-05-30 06:54:58 +00:00
logger.info(f'Reading pairs file "{pairs_file}".')
2019-05-29 18:57:14 +00:00
# Download pairs from the pairs file if no config is specified
# or if pairs file is specified explicitely
if not pairs_file.exists():
sys.exit(f'No pairs file found with path "{pairs_file}".')
2018-04-22 07:56:49 +00:00
2019-05-29 18:57:14 +00:00
with pairs_file.open() as file:
pairs = list(set(json.load(file)))
2019-05-29 18:57:14 +00:00
pairs.sort()
2018-06-24 17:52:12 +00:00
timerange = TimeRange()
if args.days:
2018-06-24 17:52:12 +00:00
time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
timerange = arguments.parse_timerange(f'{time_since}-')
2019-05-30 06:54:58 +00:00
logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}')
pairs_not_available = []
2019-05-29 18:57:14 +00:00
try:
# Init exchange
exchange = Exchange(config)
for pair in pairs:
if pair not in exchange._api.markets:
pairs_not_available.append(pair)
2019-05-30 08:07:31 +00:00
logger.info(f"Skipping pair {pair}...")
2019-05-29 18:57:14 +00:00
continue
for ticker_interval in timeframes:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
dl_file = dl_path.joinpath(filename)
if args.erase and dl_file.exists():
2019-05-30 08:07:31 +00:00
logger.info(
f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
2019-05-29 18:57:14 +00:00
dl_file.unlink()
2019-05-30 08:07:31 +00:00
logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
2019-05-29 18:57:14 +00:00
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, ticker_interval=str(ticker_interval),
2019-05-29 18:57:14 +00:00
timerange=timerange)
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
2019-05-30 08:07:31 +00:00
logger.info(
f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {config['exchange']['name']}.")