adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #622 from gcarq/fix/dl-testdata

Browse Source 
fix download testdata
This commit is contained in:
Michael Egger 2018-05-02 22:06:43 +02:00 committed by GitHub
commit 90a107393a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 95 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
freqtrade/arguments.py
View File

@ -240,7 +240,7 @@ class Arguments(object):
def scripts_options(self) -> None:
"""
Parses given arguments for plot scripts.
Parses given arguments for scripts.
"""
self.parser.add_argument(
'-p', '--pair',
@ -248,3 +248,20 @@ class Arguments(object):
dest='pair',
default=None
)
def testdata_dl_options(self) -> None:
"""
Parses given arguments for testdata download
"""
self.parser.add_argument(
'--pairs-file',
help='File containing a list of pairs to download',
dest='pairs_file',
default=None
)
self.parser.add_argument(
'--export',
help='Export files to given dir',
dest='export',
default=None)

43
freqtrade/exchange/__init__.py
View File

@ -47,6 +47,32 @@ def retrier(f):
return wrapper
def init_ccxt(exchange_config: dict) -> ccxt.Exchange:
"""
Initialize ccxt with given config and return valid
ccxt instance.
:param config: config to use
:return: ccxt
"""
# Find matching class for the given exchange name
name = exchange_config['name']
if name not in ccxt.exchanges:
raise OperationalException('Exchange {} is not supported'.format(name))
try:
api = getattr(ccxt, name.lower())({
'apiKey': exchange_config.get('key'),
'secret': exchange_config.get('secret'),
'password': exchange_config.get('password'),
'uid': exchange_config.get('uid', ''),
'enableRateLimit': True,
})
except (KeyError, AttributeError):
raise OperationalException('Exchange {} is not supported'.format(name))
return api
def init(config: dict) -> None:
"""
Initializes this module with the given config,
@ -63,22 +89,7 @@ def init(config: dict) -> None:
logger.info('Instance is running with dry_run enabled')
exchange_config = config['exchange']
# Find matching class for the given exchange name
name = exchange_config['name']
if name not in ccxt.exchanges:
raise OperationalException('Exchange {} is not supported'.format(name))
try:
_API = getattr(ccxt, name.lower())({
'apiKey': exchange_config.get('key'),
'secret': exchange_config.get('secret'),
'password': exchange_config.get('password'),
'uid': exchange_config.get('uid'),
'enableRateLimit': True,
})
except (KeyError, AttributeError):
raise OperationalException('Exchange {} is not supported'.format(name))
_API = init_ccxt(exchange_config)
logger.info('Using Exchange "%s"', get_name())

2
freqtrade/misc.py
View File

@ -71,6 +71,8 @@ def file_dump_json(filename, data, is_zip=False) -> None:
:param data: JSON Data to save
:return:
"""
print(f'dumping json to "{filename}"')
if is_zip:
if not filename.endswith('.gz'):
filename = filename + '.gz'

7
freqtrade/tests/conftest.py
View File

@ -74,10 +74,9 @@ def default_conf():
"secret": "secret",
"pair_whitelist": [
"ETH/BTC",
"TKN/BTC",
"TRST/BTC",
"SWT/BTC",
"BCC/BTC"
"LTC/BTC",
"XRP/BTC",
"NEO/BTC"
]
},
"telegram": {

2
freqtrade/tests/exchange/test_exchange.py
View File

@ -45,7 +45,7 @@ def test_init_exception(default_conf):
def test_validate_pairs(default_conf, mocker):
api_mock = MagicMock()
api_mock.load_markets = MagicMock(return_value={
'ETH/BTC': '', 'TKN/BTC': '', 'TRST/BTC': '', 'SWT/BTC': '', 'BCC/BTC': ''
'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': ''
})
id_mock = PropertyMock(return_value='test_exchange')
type(api_mock).id = id_mock

35
freqtrade/tests/testdata/download_backtest_data.py vendored
View File

@ -1,35 +0,0 @@
#!/usr/bin/env python3
"""This script generate json data from bittrex"""
import json
import sys
from freqtrade import (exchange, arguments, misc)
from freqtrade.exchange import ccxt
BASE_PATH = 'freqtrade/tests/testdata'
arguments = arguments.Arguments(sys.argv[1:], 'download utility')
arguments.scripts_options()
args = arguments.parse_args()
TICKER_INTERVALS = ['1m', '5m']
PAIRS = []
if args.pair:
with open(args.pair) as file:
PAIRS = json.load(file)
PAIRS = list(set(PAIRS))
print('About to download pairs:', PAIRS)
# Init Bittrex exchange
exchange._API = ccxt.bittrex({'key': '', 'secret': ''})
for pair in PAIRS:
for tick_interval in TICKER_INTERVALS:
print('downloading pair %s, interval %s' % (pair, tick_interval))
data = exchange.get_ticker_history(pair, tick_interval)
pair_print = pair.replace('/', '_')
filename = '{}-{}.json'.format(pair_print, tick_interval)
misc.file_dump_json(filename, data)

5
scripts/convert_backtestdata.py
View File

@ -69,7 +69,7 @@ def parse_old_backtest_data(ticker) -> DataFrame:
.rename(columns=columns)
if 'BV' in frame:
frame.drop('BV', 1, inplace=True)
if not 'date' in frame:
if 'date' not in frame:
logger.warning("Date not in frame - probably not a Ticker file")
return None
frame.sort_values('date', inplace=True)
@ -136,7 +136,8 @@ def convert_main(args: Namespace) -> None:
if ret_integer:
minutes = int(ret_integer.group(0))
interval = str(minutes) + 'm' # default to adding 'm' to end of minutes for new interval name
# default to adding 'm' to end of minutes for new interval name
interval = str(minutes) + 'm'
# but check if there is a mapping between int and string also
for str_interval, minutes_interval in Constants.TICKER_INTERVAL_MINUTES.items():
if minutes_interval == minutes:

41
scripts/download_backtest_data.py Executable file
View File

@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""This script generate json data from bittrex"""
import json
import sys
import os
from freqtrade import (exchange, arguments, misc)
DEFAULT_DL_PATH = 'freqtrade/tests/testdata'
arguments = arguments.Arguments(sys.argv[1:], 'download utility')
arguments.testdata_dl_options()
args = arguments.parse_args()
TICKER_INTERVALS = ['1m', '5m']
PAIRS = []
if args.pairs_file:
with open(args.pairs_file) as file:
PAIRS = json.load(file)
PAIRS = list(set(PAIRS))
dl_path = DEFAULT_DL_PATH
if args.export and os.path.exists(args.export):
dl_path = args.export
print(f'About to download pairs: {PAIRS} to {dl_path}')
# Init Bittrex exchange
exchange._API = exchange.init_ccxt({'key': '',
'secret': '',
'name': 'bittrex'})
for pair in PAIRS:
for tick_interval in TICKER_INTERVALS:
print(f'downloading pair {pair}, interval {tick_interval}')
data = exchange.get_ticker_history(pair, tick_interval)
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{tick_interval}.json'
misc.file_dump_json(os.path.join(dl_path, filename), data)