Merge pull request #622 from gcarq/fix/dl-testdata

fix download testdata

freqtrade/arguments.py

@@ -240,7 +240,7 @@ class Arguments(object):
 
     def scripts_options(self) -> None:
         """
-        Parses given arguments for plot scripts.
+        Parses given arguments for scripts.
         """
         self.parser.add_argument(
             '-p', '--pair',
@@ -248,3 +248,20 @@ class Arguments(object):
             dest='pair',
             default=None
         )
+
+    def testdata_dl_options(self) -> None:
+        """
+        Parses given arguments for testdata download
+        """
+        self.parser.add_argument(
+            '--pairs-file',
+            help='File containing a list of pairs to download',
+            dest='pairs_file',
+            default=None
+        )
+
+        self.parser.add_argument(
+            '--export',
+            help='Export files to given dir',
+            dest='export',
+            default=None)

freqtrade/exchange/__init__.py

@@ -47,6 +47,32 @@ def retrier(f):
     return wrapper
 
 
+def init_ccxt(exchange_config: dict) -> ccxt.Exchange:
+    """
+    Initialize ccxt with given config and return valid
+    ccxt instance.
+    :param config: config to use
+    :return: ccxt
+    """
+    # Find matching class for the given exchange name
+    name = exchange_config['name']
+
+    if name not in ccxt.exchanges:
+        raise OperationalException('Exchange {} is not supported'.format(name))
+    try:
+        api = getattr(ccxt, name.lower())({
+            'apiKey': exchange_config.get('key'),
+            'secret': exchange_config.get('secret'),
+            'password': exchange_config.get('password'),
+            'uid': exchange_config.get('uid', ''),
+            'enableRateLimit': True,
+        })
+    except (KeyError, AttributeError):
+        raise OperationalException('Exchange {} is not supported'.format(name))
+
+    return api
+
+
 def init(config: dict) -> None:
     """
     Initializes this module with the given config,
@@ -63,22 +89,7 @@ def init(config: dict) -> None:
         logger.info('Instance is running with dry_run enabled')
 
     exchange_config = config['exchange']
-
+    _API = init_ccxt(exchange_config)
-    # Find matching class for the given exchange name
-    name = exchange_config['name']
-
-    if name not in ccxt.exchanges:
-        raise OperationalException('Exchange {} is not supported'.format(name))
-    try:
-        _API = getattr(ccxt, name.lower())({
-            'apiKey': exchange_config.get('key'),
-            'secret': exchange_config.get('secret'),
-            'password': exchange_config.get('password'),
-            'uid': exchange_config.get('uid'),
-            'enableRateLimit': True,
-        })
-    except (KeyError, AttributeError):
-        raise OperationalException('Exchange {} is not supported'.format(name))
 
     logger.info('Using Exchange "%s"', get_name())
 

freqtrade/misc.py

@@ -71,6 +71,8 @@ def file_dump_json(filename, data, is_zip=False) -> None:
     :param data: JSON Data to save
     :return:
     """
+    print(f'dumping json to "{filename}"')
+
     if is_zip:
         if not filename.endswith('.gz'):
             filename = filename + '.gz'

freqtrade/tests/conftest.py

@@ -74,10 +74,9 @@ def default_conf():
             "secret": "secret",
             "pair_whitelist": [
                 "ETH/BTC",
-                "TKN/BTC",
+                "LTC/BTC",
-                "TRST/BTC",
+                "XRP/BTC",
-                "SWT/BTC",
+                "NEO/BTC"
-                "BCC/BTC"
             ]
         },
         "telegram": {

freqtrade/tests/exchange/test_exchange.py

@@ -45,7 +45,7 @@ def test_init_exception(default_conf):
 def test_validate_pairs(default_conf, mocker):
     api_mock = MagicMock()
     api_mock.load_markets = MagicMock(return_value={
-        'ETH/BTC': '', 'TKN/BTC': '', 'TRST/BTC': '', 'SWT/BTC': '', 'BCC/BTC': ''
+        'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': ''
     })
     id_mock = PropertyMock(return_value='test_exchange')
     type(api_mock).id = id_mock

freqtrade/tests/testdata/download_backtest_data.py

@@ -1,35 +0,0 @@
-#!/usr/bin/env python3
-
-"""This script generate json data from bittrex"""
-import json
-import sys
-
-from freqtrade import (exchange, arguments, misc)
-from freqtrade.exchange import ccxt
-
-BASE_PATH = 'freqtrade/tests/testdata'
-
-arguments = arguments.Arguments(sys.argv[1:], 'download utility')
-arguments.scripts_options()
-args = arguments.parse_args()
-
-TICKER_INTERVALS = ['1m', '5m']
-PAIRS = []
-
-if args.pair:
-    with open(args.pair) as file:
-        PAIRS = json.load(file)
-PAIRS = list(set(PAIRS))
-
-print('About to download pairs:', PAIRS)
-
-# Init Bittrex exchange
-exchange._API = ccxt.bittrex({'key': '', 'secret': ''})
-
-for pair in PAIRS:
-    for tick_interval in TICKER_INTERVALS:
-        print('downloading pair %s, interval %s' % (pair, tick_interval))
-        data = exchange.get_ticker_history(pair, tick_interval)
-        pair_print = pair.replace('/', '_')
-        filename = '{}-{}.json'.format(pair_print, tick_interval)
-        misc.file_dump_json(filename, data)

scripts/convert_backtestdata.py

@@ -69,7 +69,7 @@ def parse_old_backtest_data(ticker) -> DataFrame:
         .rename(columns=columns)
     if 'BV' in frame:
         frame.drop('BV', 1, inplace=True)
-    if not 'date' in frame:
+    if 'date' not in frame:
         logger.warning("Date not in frame - probably not a Ticker file")
         return None
     frame.sort_values('date', inplace=True)
@@ -136,7 +136,8 @@ def convert_main(args: Namespace) -> None:
 
             if ret_integer:
                 minutes = int(ret_integer.group(0))
-                interval = str(minutes) + 'm'  # default to adding 'm' to end of minutes for new interval name
+                # default to adding 'm' to end of minutes for new interval name
+                interval = str(minutes) + 'm'
                 # but check if there is a mapping between int and string also
                 for str_interval, minutes_interval in Constants.TICKER_INTERVAL_MINUTES.items():
                     if minutes_interval == minutes:

scripts/download_backtest_data.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+"""This script generate json data from bittrex"""
+import json
+import sys
+import os
+
+from freqtrade import (exchange, arguments, misc)
+
+DEFAULT_DL_PATH = 'freqtrade/tests/testdata'
+
+arguments = arguments.Arguments(sys.argv[1:], 'download utility')
+arguments.testdata_dl_options()
+args = arguments.parse_args()
+
+TICKER_INTERVALS = ['1m', '5m']
+PAIRS = []
+
+if args.pairs_file:
+    with open(args.pairs_file) as file:
+        PAIRS = json.load(file)
+PAIRS = list(set(PAIRS))
+
+dl_path = DEFAULT_DL_PATH
+if args.export and os.path.exists(args.export):
+    dl_path = args.export
+
+print(f'About to download pairs: {PAIRS} to {dl_path}')
+
+# Init Bittrex exchange
+exchange._API = exchange.init_ccxt({'key': '',
+                                    'secret': '',
+                                    'name': 'bittrex'})
+
+for pair in PAIRS:
+    for tick_interval in TICKER_INTERVALS:
+        print(f'downloading pair {pair}, interval {tick_interval}')
+        data = exchange.get_ticker_history(pair, tick_interval)
+        pair_print = pair.replace('/', '_')
+        filename = f'{pair_print}-{tick_interval}.json'
+        misc.file_dump_json(os.path.join(dl_path, filename), data)