stable/scripts/download_backtest_data.py

114 lines
3.2 KiB
Python
Raw Normal View History

2018-04-22 07:56:49 +00:00
#!/usr/bin/env python3
"""
This script generates json data
"""
2018-04-22 07:56:49 +00:00
import json
import sys
2018-06-24 17:52:12 +00:00
from pathlib import Path
import arrow
from typing import Any, Dict
from freqtrade.arguments import Arguments
2018-06-24 17:52:12 +00:00
from freqtrade.arguments import TimeRange
from freqtrade.exchange import Exchange
from freqtrade.data.history import download_pair_history
from freqtrade.configuration import Configuration, set_loggers
from freqtrade.misc import deep_merge_dicts
2018-06-24 17:52:12 +00:00
import logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
)
set_loggers(0)
2018-04-22 07:56:49 +00:00
DEFAULT_DL_PATH = 'user_data/data'
2018-04-22 07:56:49 +00:00
arguments = Arguments(sys.argv[1:], 'download utility')
2018-04-22 07:56:49 +00:00
arguments.testdata_dl_options()
args = arguments.parse_args()
timeframes = args.timeframes
2018-04-22 07:56:49 +00:00
if args.config:
configuration = Configuration(args)
config: Dict[str, Any] = {}
# Now expecting a list of config filenames here, not a string
for path in args.config:
print(f"Using config: {path}...")
# Merge config options, overwriting old values
config = deep_merge_dicts(configuration._load_config_file(path), config)
config['stake_currency'] = ''
# Ensure we do not use Exchange credentials
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
else:
config = {
'stake_currency': '',
'dry_run': True,
'exchange': {
'name': args.exchange,
'key': '',
'secret': '',
'pair_whitelist': [],
'ccxt_async_config': {
'enableRateLimit': False
}
}
}
dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name'])
2018-06-04 09:58:35 +00:00
if args.export:
2018-06-24 17:52:12 +00:00
dl_path = Path(args.export)
2018-06-04 09:58:35 +00:00
2018-06-24 17:52:12 +00:00
if not dl_path.is_dir():
sys.exit(f'Directory {dl_path} does not exist.')
2018-06-24 17:52:12 +00:00
pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
if not pairs_file.exists():
sys.exit(f'No pairs file found with path {pairs_file}.')
2018-04-22 07:56:49 +00:00
2018-06-24 17:52:12 +00:00
with pairs_file.open() as file:
PAIRS = list(set(json.load(file)))
2018-04-22 07:56:49 +00:00
PAIRS.sort()
2018-06-24 17:52:12 +00:00
timerange = TimeRange()
if args.days:
2018-06-24 17:52:12 +00:00
time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
timerange = arguments.parse_timerange(f'{time_since}-')
2018-04-22 07:56:49 +00:00
print(f'About to download pairs: {PAIRS} to {dl_path}')
# Init exchange
exchange = Exchange(config)
pairs_not_available = []
2018-04-22 07:56:49 +00:00
for pair in PAIRS:
if pair not in exchange._api.markets:
pairs_not_available.append(pair)
print(f"skipping pair {pair}")
continue
for ticker_interval in timeframes:
2018-04-22 07:56:49 +00:00
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
2018-06-24 17:52:12 +00:00
dl_file = dl_path.joinpath(filename)
if args.erase and dl_file.exists():
print(f'Deleting existing data for pair {pair}, interval {ticker_interval}')
2018-06-24 17:52:12 +00:00
dl_file.unlink()
print(f'downloading pair {pair}, interval {ticker_interval}')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair,
ticker_interval=ticker_interval,
timerange=timerange)
if pairs_not_available:
print(f"Pairs [{','.join(pairs_not_available)}] not availble.")