2018-04-22 07:56:49 +00:00
|
|
|
#!/usr/bin/env python3
|
2019-04-10 21:59:53 +00:00
|
|
|
"""
|
2019-05-29 18:57:14 +00:00
|
|
|
This script generates json files with pairs history data
|
2019-04-10 21:59:53 +00:00
|
|
|
"""
|
2019-05-29 18:57:14 +00:00
|
|
|
import arrow
|
2018-04-22 07:56:49 +00:00
|
|
|
import json
|
|
|
|
import sys
|
2018-06-24 17:52:12 +00:00
|
|
|
from pathlib import Path
|
2019-05-29 18:57:14 +00:00
|
|
|
from typing import Any, Dict, List
|
2018-04-27 21:16:34 +00:00
|
|
|
|
2019-07-11 18:23:23 +00:00
|
|
|
from freqtrade.configuration import Arguments, TimeRange
|
2019-05-29 18:57:14 +00:00
|
|
|
from freqtrade.configuration import Configuration
|
2019-07-14 22:55:35 +00:00
|
|
|
from freqtrade.configuration.arguments import ARGS_DOWNLOADER
|
2019-07-11 18:23:23 +00:00
|
|
|
from freqtrade.configuration.check_exchange import check_exchange
|
2019-08-10 12:14:37 +00:00
|
|
|
from freqtrade.configuration.load_config import load_config_file
|
2018-12-16 09:33:08 +00:00
|
|
|
from freqtrade.data.history import download_pair_history
|
2019-05-29 18:57:14 +00:00
|
|
|
from freqtrade.exchange import Exchange
|
2019-02-20 14:00:35 +00:00
|
|
|
from freqtrade.misc import deep_merge_dicts
|
2018-06-24 17:52:12 +00:00
|
|
|
|
2018-10-04 18:38:30 +00:00
|
|
|
import logging
|
2019-05-29 18:57:14 +00:00
|
|
|
|
|
|
|
logger = logging.getLogger('download_backtest_data')
|
2018-04-22 07:56:49 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
# Do not read the default config if config is not specified
|
|
|
|
# in the command line options explicitely
|
2019-07-18 14:55:35 +00:00
|
|
|
arguments = Arguments(sys.argv[1:], 'Download backtest data',
|
|
|
|
no_default_config=True)
|
|
|
|
arguments._build_args(optionlist=ARGS_DOWNLOADER)
|
|
|
|
args = arguments._parse_args()
|
2018-04-22 07:56:49 +00:00
|
|
|
|
2019-06-11 07:09:30 +00:00
|
|
|
# Use bittrex as default exchange
|
|
|
|
exchange_name = args.exchange or 'bittrex'
|
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
pairs: List = []
|
2018-04-22 07:56:49 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
configuration = Configuration(args)
|
|
|
|
config: Dict[str, Any] = {}
|
2019-02-20 14:00:35 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
if args.config:
|
2019-02-20 14:00:35 +00:00
|
|
|
# Now expecting a list of config filenames here, not a string
|
|
|
|
for path in args.config:
|
2019-05-30 06:54:58 +00:00
|
|
|
logger.info(f"Using config: {path}...")
|
2019-02-20 14:00:35 +00:00
|
|
|
# Merge config options, overwriting old values
|
2019-08-10 12:14:37 +00:00
|
|
|
config = deep_merge_dicts(load_config_file(path), config)
|
2019-02-20 14:00:35 +00:00
|
|
|
|
2018-12-27 13:29:26 +00:00
|
|
|
config['stake_currency'] = ''
|
2018-12-25 12:14:28 +00:00
|
|
|
# Ensure we do not use Exchange credentials
|
2019-05-29 18:57:14 +00:00
|
|
|
config['exchange']['dry_run'] = True
|
2018-12-25 12:14:28 +00:00
|
|
|
config['exchange']['key'] = ''
|
|
|
|
config['exchange']['secret'] = ''
|
2019-05-29 18:57:14 +00:00
|
|
|
|
|
|
|
pairs = config['exchange']['pair_whitelist']
|
2019-06-13 18:26:47 +00:00
|
|
|
|
2019-06-13 23:58:34 +00:00
|
|
|
if config.get('ticker_interval'):
|
|
|
|
timeframes = args.timeframes or [config.get('ticker_interval')]
|
|
|
|
else:
|
|
|
|
timeframes = args.timeframes or ['1m', '5m']
|
2019-05-29 18:57:14 +00:00
|
|
|
|
2018-12-25 12:14:28 +00:00
|
|
|
else:
|
2019-04-10 21:59:53 +00:00
|
|
|
config = {
|
|
|
|
'stake_currency': '',
|
|
|
|
'dry_run': True,
|
|
|
|
'exchange': {
|
2019-06-11 07:09:30 +00:00
|
|
|
'name': exchange_name,
|
2019-04-10 21:59:53 +00:00
|
|
|
'key': '',
|
|
|
|
'secret': '',
|
|
|
|
'pair_whitelist': [],
|
|
|
|
'ccxt_async_config': {
|
2019-04-16 17:54:04 +00:00
|
|
|
'enableRateLimit': True,
|
|
|
|
'rateLimit': 200
|
2019-04-10 21:59:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-06-13 23:58:34 +00:00
|
|
|
timeframes = args.timeframes or ['1m', '5m']
|
2019-06-13 18:26:47 +00:00
|
|
|
|
2019-07-16 20:00:19 +00:00
|
|
|
configuration._process_logging_options(config)
|
2019-06-11 07:09:30 +00:00
|
|
|
|
|
|
|
if args.config and args.exchange:
|
2019-06-11 07:21:59 +00:00
|
|
|
logger.warning("The --exchange option is ignored, "
|
|
|
|
"using exchange settings from the configuration file.")
|
2019-06-11 07:09:30 +00:00
|
|
|
|
|
|
|
# Check if the exchange set by the user is supported
|
2019-07-11 18:23:23 +00:00
|
|
|
check_exchange(config)
|
2019-06-11 07:09:30 +00:00
|
|
|
|
2019-07-16 20:00:19 +00:00
|
|
|
configuration._process_datadir_options(config)
|
2018-12-25 12:14:28 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
dl_path = Path(config['datadir'])
|
2018-06-04 09:20:17 +00:00
|
|
|
|
2018-06-24 17:52:12 +00:00
|
|
|
pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
|
2018-04-22 07:56:49 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
if not pairs or args.pairs_file:
|
2019-05-30 06:54:58 +00:00
|
|
|
logger.info(f'Reading pairs file "{pairs_file}".')
|
2019-05-29 18:57:14 +00:00
|
|
|
# Download pairs from the pairs file if no config is specified
|
|
|
|
# or if pairs file is specified explicitely
|
|
|
|
if not pairs_file.exists():
|
|
|
|
sys.exit(f'No pairs file found with path "{pairs_file}".')
|
2018-04-22 07:56:49 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
with pairs_file.open() as file:
|
|
|
|
pairs = list(set(json.load(file)))
|
2018-06-11 19:10:57 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
pairs.sort()
|
2018-06-24 17:52:12 +00:00
|
|
|
|
|
|
|
timerange = TimeRange()
|
2018-04-27 21:16:34 +00:00
|
|
|
if args.days:
|
2018-06-24 17:52:12 +00:00
|
|
|
time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
|
2019-08-14 08:07:32 +00:00
|
|
|
timerange = TimeRange.parse_timerange(f'{time_since}-')
|
2018-04-27 21:16:34 +00:00
|
|
|
|
2019-05-30 06:54:58 +00:00
|
|
|
logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}')
|
2018-04-27 21:16:34 +00:00
|
|
|
|
2018-06-11 19:10:57 +00:00
|
|
|
pairs_not_available = []
|
2018-04-27 21:16:34 +00:00
|
|
|
|
2019-05-29 18:57:14 +00:00
|
|
|
try:
|
|
|
|
# Init exchange
|
|
|
|
exchange = Exchange(config)
|
|
|
|
|
|
|
|
for pair in pairs:
|
|
|
|
if pair not in exchange._api.markets:
|
|
|
|
pairs_not_available.append(pair)
|
2019-05-30 08:07:31 +00:00
|
|
|
logger.info(f"Skipping pair {pair}...")
|
2019-05-29 18:57:14 +00:00
|
|
|
continue
|
|
|
|
for ticker_interval in timeframes:
|
|
|
|
pair_print = pair.replace('/', '_')
|
|
|
|
filename = f'{pair_print}-{ticker_interval}.json'
|
|
|
|
dl_file = dl_path.joinpath(filename)
|
|
|
|
if args.erase and dl_file.exists():
|
2019-05-30 08:07:31 +00:00
|
|
|
logger.info(
|
|
|
|
f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
|
2019-05-29 18:57:14 +00:00
|
|
|
dl_file.unlink()
|
|
|
|
|
2019-05-30 08:07:31 +00:00
|
|
|
logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
|
2019-05-29 18:57:14 +00:00
|
|
|
download_pair_history(datadir=dl_path, exchange=exchange,
|
2019-06-13 18:37:17 +00:00
|
|
|
pair=pair, ticker_interval=str(ticker_interval),
|
2019-05-29 18:57:14 +00:00
|
|
|
timerange=timerange)
|
|
|
|
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
sys.exit("SIGINT received, aborting ...")
|
|
|
|
|
|
|
|
finally:
|
|
|
|
if pairs_not_available:
|
2019-05-30 08:07:31 +00:00
|
|
|
logger.info(
|
|
|
|
f"Pairs [{','.join(pairs_not_available)}] not available "
|
|
|
|
f"on exchange {config['exchange']['name']}.")
|