2018-04-22 07:56:49 +00:00
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
2018-08-10 09:08:28 +00:00
|
|
|
|
"""This script generate json data"""
|
2018-04-22 07:56:49 +00:00
|
|
|
|
import json
|
|
|
|
|
import sys
|
2018-06-24 17:52:12 +00:00
|
|
|
|
from pathlib import Path
|
2018-04-27 21:16:34 +00:00
|
|
|
|
import arrow
|
|
|
|
|
|
2018-06-24 17:52:12 +00:00
|
|
|
|
from freqtrade import arguments
|
|
|
|
|
from freqtrade.arguments import TimeRange
|
2018-06-18 20:28:51 +00:00
|
|
|
|
from freqtrade.exchange import Exchange
|
2018-12-13 05:34:37 +00:00
|
|
|
|
from freqtrade.data.history import download_backtesting_testdata
|
2018-10-04 18:38:30 +00:00
|
|
|
|
from freqtrade.configuration import set_loggers
|
2018-06-24 17:52:12 +00:00
|
|
|
|
|
2018-10-04 18:38:30 +00:00
|
|
|
|
import logging
|
|
|
|
|
logging.basicConfig(
|
|
|
|
|
level=logging.INFO,
|
|
|
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
|
|
|
)
|
|
|
|
|
set_loggers(0)
|
2018-04-22 07:56:49 +00:00
|
|
|
|
|
2018-06-04 09:20:17 +00:00
|
|
|
|
DEFAULT_DL_PATH = 'user_data/data'
|
2018-04-22 07:56:49 +00:00
|
|
|
|
|
|
|
|
|
arguments = arguments.Arguments(sys.argv[1:], 'download utility')
|
|
|
|
|
arguments.testdata_dl_options()
|
|
|
|
|
args = arguments.parse_args()
|
|
|
|
|
|
2018-06-04 10:31:52 +00:00
|
|
|
|
timeframes = args.timeframes
|
2018-04-22 07:56:49 +00:00
|
|
|
|
|
2018-06-24 17:52:12 +00:00
|
|
|
|
dl_path = Path(DEFAULT_DL_PATH).joinpath(args.exchange)
|
2018-06-04 09:58:35 +00:00
|
|
|
|
if args.export:
|
2018-06-24 17:52:12 +00:00
|
|
|
|
dl_path = Path(args.export)
|
2018-06-04 09:58:35 +00:00
|
|
|
|
|
2018-06-24 17:52:12 +00:00
|
|
|
|
if not dl_path.is_dir():
|
2018-06-04 09:20:17 +00:00
|
|
|
|
sys.exit(f'Directory {dl_path} does not exist.')
|
|
|
|
|
|
2018-06-24 17:52:12 +00:00
|
|
|
|
pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
|
|
|
|
|
if not pairs_file.exists():
|
2018-06-04 09:20:17 +00:00
|
|
|
|
sys.exit(f'No pairs file found with path {pairs_file}.')
|
2018-04-22 07:56:49 +00:00
|
|
|
|
|
2018-06-24 17:52:12 +00:00
|
|
|
|
with pairs_file.open() as file:
|
2018-06-04 09:20:17 +00:00
|
|
|
|
PAIRS = list(set(json.load(file)))
|
2018-04-22 07:56:49 +00:00
|
|
|
|
|
2018-06-11 19:10:57 +00:00
|
|
|
|
PAIRS.sort()
|
|
|
|
|
|
2018-06-24 17:52:12 +00:00
|
|
|
|
|
|
|
|
|
timerange = TimeRange()
|
2018-04-27 21:16:34 +00:00
|
|
|
|
if args.days:
|
2018-06-24 17:52:12 +00:00
|
|
|
|
time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
|
|
|
|
|
timerange = arguments.parse_timerange(f'{time_since}-')
|
2018-04-27 21:16:34 +00:00
|
|
|
|
|
|
|
|
|
|
2018-04-22 07:56:49 +00:00
|
|
|
|
print(f'About to download pairs: {PAIRS} to {dl_path}')
|
|
|
|
|
|
2018-06-18 20:28:51 +00:00
|
|
|
|
|
2018-04-24 23:11:07 +00:00
|
|
|
|
# Init exchange
|
2018-06-18 20:28:51 +00:00
|
|
|
|
exchange = Exchange({'key': '',
|
|
|
|
|
'secret': '',
|
|
|
|
|
'stake_currency': '',
|
|
|
|
|
'dry_run': True,
|
|
|
|
|
'exchange': {
|
2018-08-10 09:08:28 +00:00
|
|
|
|
'name': args.exchange,
|
|
|
|
|
'pair_whitelist': [],
|
2018-10-04 18:38:30 +00:00
|
|
|
|
'ccxt_async_config': {
|
|
|
|
|
"enableRateLimit": False
|
|
|
|
|
}
|
2018-08-10 09:08:28 +00:00
|
|
|
|
}
|
2018-06-18 20:28:51 +00:00
|
|
|
|
})
|
2018-06-11 19:10:57 +00:00
|
|
|
|
pairs_not_available = []
|
2018-04-27 21:16:34 +00:00
|
|
|
|
|
2018-04-22 07:56:49 +00:00
|
|
|
|
for pair in PAIRS:
|
2018-06-18 20:28:51 +00:00
|
|
|
|
if pair not in exchange._api.markets:
|
2018-06-11 19:10:57 +00:00
|
|
|
|
pairs_not_available.append(pair)
|
|
|
|
|
print(f"skipping pair {pair}")
|
|
|
|
|
continue
|
2018-06-04 10:31:52 +00:00
|
|
|
|
for tick_interval in timeframes:
|
2018-04-22 07:56:49 +00:00
|
|
|
|
pair_print = pair.replace('/', '_')
|
|
|
|
|
filename = f'{pair_print}-{tick_interval}.json'
|
2018-06-24 17:52:12 +00:00
|
|
|
|
dl_file = dl_path.joinpath(filename)
|
|
|
|
|
if args.erase and dl_file.exists():
|
|
|
|
|
print(f'Deleting existing data for pair {pair}, interval {tick_interval}')
|
|
|
|
|
dl_file.unlink()
|
|
|
|
|
|
|
|
|
|
print(f'downloading pair {pair}, interval {tick_interval}')
|
|
|
|
|
download_backtesting_testdata(str(dl_path), exchange=exchange,
|
|
|
|
|
pair=pair,
|
|
|
|
|
tick_interval=tick_interval,
|
|
|
|
|
timerange=timerange)
|
2018-06-11 19:10:57 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if pairs_not_available:
|
|
|
|
|
print(f"Pairs [{','.join(pairs_not_available)}] not availble.")
|