From fded8e51173906fca63ffee2ce6d9e3622d36de5 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 22 Apr 2018 10:56:49 +0300 Subject: [PATCH 1/5] move download_backtest_data to scripts --- .../tests/testdata/download_backtest_data.py | 35 ---------------- scripts/download_backtest_data.py | 41 +++++++++++++++++++ 2 files changed, 41 insertions(+), 35 deletions(-) delete mode 100755 freqtrade/tests/testdata/download_backtest_data.py create mode 100755 scripts/download_backtest_data.py diff --git a/freqtrade/tests/testdata/download_backtest_data.py b/freqtrade/tests/testdata/download_backtest_data.py deleted file mode 100755 index 8c7d6daea..000000000 --- a/freqtrade/tests/testdata/download_backtest_data.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 - -"""This script generate json data from bittrex""" -import json -import sys - -from freqtrade import (exchange, arguments, misc) -from freqtrade.exchange import ccxt - -BASE_PATH = 'freqtrade/tests/testdata' - -arguments = arguments.Arguments(sys.argv[1:], 'download utility') -arguments.scripts_options() -args = arguments.parse_args() - -TICKER_INTERVALS = ['1m', '5m'] -PAIRS = [] - -if args.pair: - with open(args.pair) as file: - PAIRS = json.load(file) -PAIRS = list(set(PAIRS)) - -print('About to download pairs:', PAIRS) - -# Init Bittrex exchange -exchange._API = ccxt.bittrex({'key': '', 'secret': ''}) - -for pair in PAIRS: - for tick_interval in TICKER_INTERVALS: - print('downloading pair %s, interval %s' % (pair, tick_interval)) - data = exchange.get_ticker_history(pair, tick_interval) - pair_print = pair.replace('/', '_') - filename = '{}-{}.json'.format(pair_print, tick_interval) - misc.file_dump_json(filename, data) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py new file mode 100755 index 000000000..8230c1349 --- /dev/null +++ b/scripts/download_backtest_data.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +"""This script generate json data from bittrex""" +import json +import sys +import os + +from freqtrade import (exchange, arguments, misc) + +DEFAULT_DL_PATH = 'freqtrade/tests/testdata' + +arguments = arguments.Arguments(sys.argv[1:], 'download utility') +arguments.testdata_dl_options() +args = arguments.parse_args() + +TICKER_INTERVALS = ['1m', '5m'] +PAIRS = [] + +if args.pairs_file: + with open(args.pairs_file) as file: + PAIRS = json.load(file) +PAIRS = list(set(PAIRS)) + +dl_path = DEFAULT_DL_PATH +if args.export and os.path.exists(args.export): + dl_path = args.export + +print(f'About to download pairs: {PAIRS} to {dl_path}') + +# Init Bittrex exchange +exchange._API = exchange.init_ccxt({'key': '', + 'secret': '', + 'name': 'bittrex'}) + +for pair in PAIRS: + for tick_interval in TICKER_INTERVALS: + print(f'downloading pair {pair}, interval {tick_interval}') + data = exchange.get_ticker_history(pair, tick_interval) + pair_print = pair.replace('/', '_') + filename = f'{pair_print}-{tick_interval}.json' + misc.file_dump_json(os.path.join(dl_path, filename), data) From de8db9293ce68602b1d5db00162aafd225ec0f25 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 22 Apr 2018 10:57:48 +0300 Subject: [PATCH 2/5] exchange: extract ccxt init to its own function (so that we can init ccxt from the scripts) --- freqtrade/arguments.py | 19 ++++++++++++++- freqtrade/exchange/__init__.py | 43 +++++++++++++++++++++------------- freqtrade/misc.py | 2 ++ 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index cf17ed096..574deb304 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -240,7 +240,7 @@ class Arguments(object): def scripts_options(self) -> None: """ - Parses given arguments for plot scripts. + Parses given arguments for scripts. """ self.parser.add_argument( '-p', '--pair', @@ -248,3 +248,20 @@ class Arguments(object): dest='pair', default=None ) + + def testdata_dl_options(self) -> None: + """ + Parses given arguments for testdata download + """ + self.parser.add_argument( + '--pairs-file', + help='File containing a list of pairs to download', + dest='pairs_file', + default=None + ) + + self.parser.add_argument( + '--export', + help='Export files to given dir', + dest='export', + default=None) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index dffe0e1b8..91d2493cc 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -45,6 +45,32 @@ def retrier(f): return wrapper +def init_ccxt(exchange_config: dict) -> ccxt: + """ + Initialize ccxt with given config and return valid + ccxt instance. + :param config: config to use + :return: ccxt + """ + # Find matching class for the given exchange name + name = exchange_config['name'] + + if name not in ccxt.exchanges: + raise OperationalException('Exchange {} is not supported'.format(name)) + try: + api = getattr(ccxt, name.lower())({ + 'apiKey': exchange_config.get('key'), + 'secret': exchange_config.get('secret'), + 'password': exchange_config.get('password'), + 'uid': exchange_config.get('uid' ''), + 'enableRateLimit': True, + }) + except (KeyError, AttributeError): + raise OperationalException('Exchange {} is not supported'.format(name)) + + return api + + def init(config: dict) -> None: """ Initializes this module with the given config, @@ -61,22 +87,7 @@ def init(config: dict) -> None: logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] - - # Find matching class for the given exchange name - name = exchange_config['name'] - - if name not in ccxt.exchanges: - raise OperationalException('Exchange {} is not supported'.format(name)) - try: - _API = getattr(ccxt, name.lower())({ - 'apiKey': exchange_config.get('key'), - 'secret': exchange_config.get('secret'), - 'password': exchange_config.get('password'), - 'uid': exchange_config.get('uid'), - 'enableRateLimit': True, - }) - except (KeyError, AttributeError): - raise OperationalException('Exchange {} is not supported'.format(name)) + _API = init_ccxt(exchange_config) logger.info('Using Exchange "%s"', get_name()) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 7546dba8f..225fb32df 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -71,6 +71,8 @@ def file_dump_json(filename, data, is_zip=False) -> None: :param data: JSON Data to save :return: """ + print(f'dumping json to "{filename}"') + if is_zip: if not filename.endswith('.gz'): filename = filename + '.gz' From 53e76a89acb1df5c108ad950ce4c78dfb528df43 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 22 Apr 2018 11:00:51 +0300 Subject: [PATCH 3/5] convert_backtestdata: flake8 fixes --- scripts/convert_backtestdata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 302170ca0..d93356cf6 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -69,7 +69,7 @@ def parse_old_backtest_data(ticker) -> DataFrame: .rename(columns=columns) if 'BV' in frame: frame.drop('BV', 1, inplace=True) - if not 'date' in frame: + if 'date' not in frame: logger.warning("Date not in frame - probably not a Ticker file") return None frame.sort_values('date', inplace=True) @@ -136,7 +136,8 @@ def convert_main(args: Namespace) -> None: if ret_integer: minutes = int(ret_integer.group(0)) - interval = str(minutes) + 'm' # default to adding 'm' to end of minutes for new interval name + # default to adding 'm' to end of minutes for new interval name + interval = str(minutes) + 'm' # but check if there is a mapping between int and string also for str_interval, minutes_interval in Constants.TICKER_INTERVAL_MINUTES.items(): if minutes_interval == minutes: From 9f1544978dfd4a4d549fb0c8b3cd97f054ab8cc8 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 22 Apr 2018 11:29:21 +0300 Subject: [PATCH 4/5] tests: use only coins that most likely are going to be in bittrex --- freqtrade/tests/conftest.py | 7 +++---- freqtrade/tests/exchange/test_exchange.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 648480bea..060073771 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -74,10 +74,9 @@ def default_conf(): "secret": "secret", "pair_whitelist": [ "ETH/BTC", - "TKN/BTC", - "TRST/BTC", - "SWT/BTC", - "BCC/BTC" + "LTC/BTC", + "XRP/BTC", + "NEO/BTC" ] }, "telegram": { diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index c2a553d3b..e959c91b6 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -44,7 +44,7 @@ def test_init_exception(default_conf): def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={ - 'ETH/BTC': '', 'TKN/BTC': '', 'TRST/BTC': '', 'SWT/BTC': '', 'BCC/BTC': '' + 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' }) id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock From 842b0c227079fba224e845e10495cb06150fcb48 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 29 Apr 2018 18:55:43 +0300 Subject: [PATCH 5/5] Exchange: fix missing comma and typehinting per PR comments --- freqtrade/exchange/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 91d2493cc..156afd5ee 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -45,7 +45,7 @@ def retrier(f): return wrapper -def init_ccxt(exchange_config: dict) -> ccxt: +def init_ccxt(exchange_config: dict) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -62,7 +62,7 @@ def init_ccxt(exchange_config: dict) -> ccxt: 'apiKey': exchange_config.get('key'), 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), - 'uid': exchange_config.get('uid' ''), + 'uid': exchange_config.get('uid', ''), 'enableRateLimit': True, }) except (KeyError, AttributeError):