adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactoring download_backtest_data.py

Browse Source
This commit is contained in:
hroff-1902 2019-05-29 21:57:14 +03:00
parent 6451feee0e
commit fb88953be3
4 changed files with 119 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
freqtrade/arguments.py
View File

@ -47,7 +47,7 @@ class Arguments(object):
return self.parsed_arg
def parse_args(self) -> argparse.Namespace:
def parse_args(self, no_default_config: bool = False) -> argparse.Namespace:
"""
Parses given arguments and returns an argparse Namespace instance.
"""
@ -55,7 +55,7 @@ class Arguments(object):
# Workaround issue in argparse with action='append' and default value
# (see https://bugs.python.org/issue16399)
if parsed_arg.config is None:
if parsed_arg.config is None and not no_default_config:
parsed_arg.config = [constants.DEFAULT_CONFIG]
return parsed_arg
@ -427,26 +427,24 @@ class Arguments(object):
default=None
)
def testdata_dl_options(self) -> None:
def download_data_options(self) -> None:
"""
Parses given arguments for testdata download
"""
self.parser.add_argument(
'--pairs-file',
help='File containing a list of pairs to download.',
dest='pairs_file',
default=None,
metavar='PATH',
'-v', '--verbose',
help='Verbose mode (-vv for more, -vvv to get all messages).',
action='count',
dest='loglevel',
default=0,
)
self.parser.add_argument(
'--export',
help='Export files to given dir.',
dest='export',
default=None,
metavar='PATH',
'--logfile',
help='Log to the file specified',
dest='logfile',
type=str,
metavar='FILE'
)
self.parser.add_argument(
'-c', '--config',
help='Specify configuration file (default: %(default)s). '
@ -456,7 +454,21 @@ class Arguments(object):
type=str,
metavar='PATH',
)
self.parser.add_argument(
'-d', '--datadir',
help='Path to backtest data.',
dest='datadir',
default=None,
type=str,
metavar='PATH',
)
self.parser.add_argument(
'--pairs-file',
help='File containing a list of pairs to download.',
dest='pairs_file',
default=None,
metavar='PATH',
)
self.parser.add_argument(
'--days',
help='Download data for given number of days.',
@ -465,7 +477,6 @@ class Arguments(object):
metavar='INT',
default=None
)
self.parser.add_argument(
'--exchange',
help='Exchange name (default: %(default)s). Only valid if no config is provided.',
@ -473,7 +484,6 @@ class Arguments(object):
type=str,
default='bittrex'
)
self.parser.add_argument(
'-t', '--timeframes',
help='Specify which tickers to download. Space separated list. \
@ -484,7 +494,6 @@ class Arguments(object):
nargs='+',
dest='timeframes',
)
self.parser.add_argument(
'--erase',
help='Clean all existing data for the selected exchange/pairs/timeframes.',

31
freqtrade/configuration.py
View File

@ -122,12 +122,11 @@ class Configuration(object):
return conf
def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
def _load_logging_config(self, config: Dict[str, Any]) -> None:
"""
Extract information for sys.argv and load common configuration
:return: configuration as dictionary
Extract information for sys.argv and load logging configuration:
the --loglevel, --logfile options
"""
# Log level
if 'loglevel' in self.args and self.args.loglevel:
config.update({'verbosity': self.args.loglevel})
@ -153,6 +152,13 @@ class Configuration(object):
set_loggers(config['verbosity'])
logger.info('Verbosity set to %s', config['verbosity'])
def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
Extract information for sys.argv and load common configuration
:return: configuration as dictionary
"""
self._load_logging_config(config)
# Support for sd_notify
if self.args.sd_notify:
config['internals'].update({'sd_notify': True})
@ -228,6 +234,17 @@ class Configuration(object):
else:
logger.info(logstring.format(config[argname]))
def _load_datadir_config(self, config: Dict[str, Any]) -> None:
"""
Extract information for sys.argv and load datadir configuration:
the --datadir option
"""
if 'datadir' in self.args and self.args.datadir:
config.update({'datadir': self._create_datadir(config, self.args.datadir)})
else:
config.update({'datadir': self._create_datadir(config, None)})
logger.info('Using data folder: %s ...', config.get('datadir'))
def _load_optimize_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
Extract information for sys.argv and load Optimize configuration
@ -263,11 +280,7 @@ class Configuration(object):
self._args_to_config(config, argname='timerange',
logstring='Parameter --timerange detected: {} ...')
if 'datadir' in self.args and self.args.datadir:
config.update({'datadir': self._create_datadir(config, self.args.datadir)})
else:
config.update({'datadir': self._create_datadir(config, None)})
logger.info('Using data folder: %s ...', config.get('datadir'))
self._load_datadir_config(config)
self._args_to_config(config, argname='refresh_pairs',
logstring='Parameter -r/--refresh-pairs-cached detected ...')

8
freqtrade/tests/test_arguments.py
View File

@ -170,18 +170,18 @@ def test_parse_args_hyperopt_custom() -> None:
assert call_args.func is not None
def test_testdata_dl_options() -> None:
def test_download_data_options() -> None:
args = [
'--pairs-file', 'file_with_pairs',
'--export', 'export/folder',
'--datadir', 'datadir/folder',
'--days', '30',
'--exchange', 'binance'
]
arguments = Arguments(args, '')
arguments.testdata_dl_options()
arguments.download_data_options()
args = arguments.parse_args()
assert args.pairs_file == 'file_with_pairs'
assert args.export == 'export/folder'
assert args.datadir == 'datadir/folder'
assert args.days == 30
assert args.exchange == 'binance'

116
scripts/download_backtest_data.py
View File

@ -1,39 +1,39 @@
#!/usr/bin/env python3
"""
This script generates json data
This script generates json files with pairs history data
"""
import arrow
import json
import sys
from pathlib import Path
import arrow
from typing import Any, Dict
from typing import Any, Dict, List
from freqtrade.arguments import Arguments
from freqtrade.arguments import TimeRange
from freqtrade.exchange import Exchange
from freqtrade.arguments import Arguments, TimeRange
from freqtrade.configuration import Configuration
from freqtrade.data.history import download_pair_history
from freqtrade.configuration import Configuration, set_loggers
from freqtrade.exchange import Exchange
from freqtrade.misc import deep_merge_dicts
import logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
)
set_loggers(0)
logger = logging.getLogger('download_backtest_data')
DEFAULT_DL_PATH = 'user_data/data'
arguments = Arguments(sys.argv[1:], 'download utility')
arguments.testdata_dl_options()
args = arguments.parse_args()
arguments.download_data_options()
# Do not read the default config if config is not specified
# in the command line options explicitely
args = arguments.parse_args(no_default_config=True)
timeframes = args.timeframes
pairs: List = []
configuration = Configuration(args)
config: Dict[str, Any] = {}
if args.config:
configuration = Configuration(args)
config: Dict[str, Any] = {}
# Now expecting a list of config filenames here, not a string
for path in args.config:
print(f"Using config: {path}...")
@ -42,9 +42,19 @@ if args.config:
config['stake_currency'] = ''
# Ensure we do not use Exchange credentials
config['exchange']['dry_run'] = True
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
if args.exchange:
config['exchange']['name'] = args.exchange
pairs = config['exchange']['pair_whitelist']
timeframes = [config['ticker_interval']]
else:
if not args.exchange:
sys.exit("No exchange specified.")
config = {
'stake_currency': '',
'dry_run': True,
@ -60,55 +70,59 @@ else:
}
}
configuration._load_logging_config(config)
configuration._load_datadir_config(config)
dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name'])
if args.export:
dl_path = Path(args.export)
if not dl_path.is_dir():
sys.exit(f'Directory {dl_path} does not exist.')
dl_path = Path(config['datadir'])
pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
if not pairs_file.exists():
sys.exit(f'No pairs file found with path {pairs_file}.')
with pairs_file.open() as file:
PAIRS = list(set(json.load(file)))
if not pairs or args.pairs_file:
print(f'Reading pairs file "{pairs_file}".')
# Download pairs from the pairs file if no config is specified
# or if pairs file is specified explicitely
if not pairs_file.exists():
sys.exit(f'No pairs file found with path "{pairs_file}".')
PAIRS.sort()
with pairs_file.open() as file:
pairs = list(set(json.load(file)))
pairs.sort()
timerange = TimeRange()
if args.days:
time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
timerange = arguments.parse_timerange(f'{time_since}-')
print(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}')
print(f'About to download pairs: {PAIRS} to {dl_path}')
# Init exchange
exchange = Exchange(config)
pairs_not_available = []
for pair in PAIRS:
if pair not in exchange._api.markets:
pairs_not_available.append(pair)
print(f"skipping pair {pair}")
continue
for ticker_interval in timeframes:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
dl_file = dl_path.joinpath(filename)
if args.erase and dl_file.exists():
print(f'Deleting existing data for pair {pair}, interval {ticker_interval}')
dl_file.unlink()
try:
# Init exchange
exchange = Exchange(config)
print(f'downloading pair {pair}, interval {ticker_interval}')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair,
ticker_interval=ticker_interval,
timerange=timerange)
for pair in pairs:
if pair not in exchange._api.markets:
pairs_not_available.append(pair)
print(f"skipping pair {pair}")
continue
for ticker_interval in timeframes:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
dl_file = dl_path.joinpath(filename)
if args.erase and dl_file.exists():
print(f'Deleting existing data for pair {pair}, interval {ticker_interval}')
dl_file.unlink()
print(f'downloading pair {pair}, interval {ticker_interval}')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, ticker_interval=ticker_interval,
timerange=timerange)
if pairs_not_available:
print(f"Pairs [{','.join(pairs_not_available)}] not availble.")
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
print(f"Pairs [{','.join(pairs_not_available)}] not availble.")