stable/scripts/download_backtest_data.py
2019-06-14 02:58:34 +03:00

142 lines
4.3 KiB
Python
Executable File

#!/usr/bin/env python3
"""
This script generates json files with pairs history data
"""
import arrow
import json
import sys
from pathlib import Path
from typing import Any, Dict, List
from freqtrade.arguments import Arguments, TimeRange
from freqtrade.configuration import Configuration
from freqtrade.data.history import download_pair_history
from freqtrade.exchange import Exchange
from freqtrade.misc import deep_merge_dicts
import logging
logger = logging.getLogger('download_backtest_data')
DEFAULT_DL_PATH = 'user_data/data'
arguments = Arguments(sys.argv[1:], 'download utility')
arguments.download_data_options()
# Do not read the default config if config is not specified
# in the command line options explicitely
args = arguments.parse_args(no_default_config=True)
# Use bittrex as default exchange
exchange_name = args.exchange or 'bittrex'
pairs: List = []
configuration = Configuration(args)
config: Dict[str, Any] = {}
if args.config:
# Now expecting a list of config filenames here, not a string
for path in args.config:
logger.info(f"Using config: {path}...")
# Merge config options, overwriting old values
config = deep_merge_dicts(configuration._load_config_file(path), config)
config['stake_currency'] = ''
# Ensure we do not use Exchange credentials
config['exchange']['dry_run'] = True
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
pairs = config['exchange']['pair_whitelist']
if config.get('ticker_interval'):
timeframes = args.timeframes or [config.get('ticker_interval')]
else:
timeframes = args.timeframes or ['1m', '5m']
else:
config = {
'stake_currency': '',
'dry_run': True,
'exchange': {
'name': exchange_name,
'key': '',
'secret': '',
'pair_whitelist': [],
'ccxt_async_config': {
'enableRateLimit': True,
'rateLimit': 200
}
}
}
timeframes = args.timeframes or ['1m', '5m']
configuration._load_logging_config(config)
if args.config and args.exchange:
logger.warning("The --exchange option is ignored, "
"using exchange settings from the configuration file.")
# Check if the exchange set by the user is supported
configuration.check_exchange(config)
configuration._load_datadir_config(config)
dl_path = Path(config['datadir'])
pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
if not pairs or args.pairs_file:
logger.info(f'Reading pairs file "{pairs_file}".')
# Download pairs from the pairs file if no config is specified
# or if pairs file is specified explicitely
if not pairs_file.exists():
sys.exit(f'No pairs file found with path "{pairs_file}".')
with pairs_file.open() as file:
pairs = list(set(json.load(file)))
pairs.sort()
timerange = TimeRange()
if args.days:
time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
timerange = arguments.parse_timerange(f'{time_since}-')
logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}')
pairs_not_available = []
try:
# Init exchange
exchange = Exchange(config)
for pair in pairs:
if pair not in exchange._api.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for ticker_interval in timeframes:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
dl_file = dl_path.joinpath(filename)
if args.erase and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, ticker_interval=str(ticker_interval),
timerange=timerange)
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
logger.info(
f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {config['exchange']['name']}.")