Merge branch 'feat/objectify-ccxt' into ccxt-objectify-pr1

freqtrade/configuration.py

@@ -5,10 +5,11 @@ This module contains the configuration class
 import json
 from argparse import Namespace
 from typing import Dict, Any
-
 from jsonschema import Draft4Validator, validate
 from jsonschema.exceptions import ValidationError, best_match
+import ccxt
 
+from freqtrade import OperationalException
 from freqtrade.constants import Constants
 from freqtrade.logger import Logger
 
@@ -100,6 +101,9 @@ class Configuration(object):
             else:
                 self.logger.info('Dry run is disabled. (--dry_run_db ignored)')
 
+        # Check if the exchange set by the user is supported
+        self.check_exchange(config)
+
         return config
 
     def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
@@ -198,3 +202,23 @@ class Configuration(object):
             self.config = self.load_config()
 
         return self.config
+
+    def check_exchange(self, config: Dict[str, Any]) -> bool:
+        """
+        Check if the exchange name in the config file is supported by Freqtrade
+        :return: True or raised an exception if the exchange if not supported
+        """
+        exchange = config.get('exchange', {}).get('name').lower()
+        if exchange not in ccxt.exchanges:
+
+            exception_msg = 'Exchange "{}" not supported.\n' \
+                            'The following exchanges are supported: {}'\
+                .format(exchange, ', '.join(ccxt.exchanges))
+
+            self.logger.critical(exception_msg)
+            raise OperationalException(
+                exception_msg
+            )
+
+        self.logger.debug('Exchange "%s" supported', exchange)
+        return True

freqtrade/exchange/__init__.py

@@ -67,7 +67,6 @@ def init(config: dict) -> None:
 
     if name not in ccxt.exchanges:
         raise OperationalException('Exchange {} is not supported'.format(name))
-
     try:
         _API = getattr(ccxt, name.lower())({
             'apiKey': exchange_config.get('key'),
@@ -75,7 +74,7 @@ def init(config: dict) -> None:
             'password': exchange_config.get('password'),
             'uid': exchange_config.get('uid'),
         })
-    except KeyError:
+    except (KeyError, AttributeError):
         raise OperationalException('Exchange {} is not supported'.format(name))
 
     logger.info('Using Exchange "%s"', get_name())
@@ -92,9 +91,6 @@ def validate_pairs(pairs: List[str]) -> None:
     :return: None
     """
 
-    if not _API.markets:
-        _API.load_markets()
-
     try:
         markets = _API.load_markets()
     except ccxt.BaseError as e:

freqtrade/misc.py

@@ -5,6 +5,7 @@ Various tool function for Freqtrade and scripts
 import json
 import logging
 import re
+import gzip
 from datetime import datetime
 from typing import Dict
 
@@ -63,15 +64,21 @@ def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray:
     return np.sort(arr, axis=0)
 
 
-def file_dump_json(filename, data) -> None:
+def file_dump_json(filename, data, is_zip=False) -> None:
     """
     Dump JSON data into a file
     :param filename: file to create
     :param data: JSON Data to save
     :return:
     """
-    with open(filename, 'w') as fp:
+    if is_zip:
-        json.dump(data, fp, default=str)
+        if not filename.endswith('.gz'):
+            filename = filename + '.gz'
+        with gzip.open(filename, 'w') as fp:
+            json.dump(data, fp, default=str)
+    else:
+        with open(filename, 'w') as fp:
+            json.dump(data, fp, default=str)
 
 
 def format_ms_time(date: str) -> str:

freqtrade/tests/exchange/test_exchange.py

@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement
 # pragma pylint: disable=protected-access
 import logging
+from copy import deepcopy
 from random import randint
 from unittest.mock import MagicMock, PropertyMock
 import ccxt
@@ -69,21 +70,40 @@ def test_validate_pairs_not_compatible(default_conf, mocker):
     })
     default_conf['stake_currency'] = 'ETH'
     mocker.patch('freqtrade.exchange._API', api_mock)
-    mocker.patch.dict('freqtrade.exchange._CONF', default_conf)
+    mocker.patch.dict('freqtrade.exchange._CONF', conf)
     with pytest.raises(OperationalException, match=r'not compatible'):
-        validate_pairs(default_conf['exchange']['pair_whitelist'])
+        validate_pairs(conf['exchange']['pair_whitelist'])
 
 
 def test_validate_pairs_exception(default_conf, mocker, caplog):
     caplog.set_level(logging.INFO)
     api_mock = MagicMock()
     api_mock.load_markets = MagicMock(side_effect=ccxt.BaseError())
+    api_mock.name = 'binance'
     mocker.patch('freqtrade.exchange._API', api_mock)
+    mocker.patch.dict('freqtrade.exchange._CONF', default_conf)
+
+    with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at binance'):
+        validate_pairs(default_conf['exchange']['pair_whitelist'])
 
     validate_pairs(default_conf['exchange']['pair_whitelist'])
     assert log_has('Unable to validate pairs (assuming they are correct). Reason: ',
                    caplog.record_tuples)
 
+def test_validate_pairs_stake_exception(default_conf, mocker, caplog):
+    caplog.set_level(logging.INFO)
+    conf = deepcopy(default_conf)
+    conf['stake_currency'] = 'ETH'
+    api_mock = MagicMock()
+    api_mock.name = 'binance'
+    mocker.patch('freqtrade.exchange._API', api_mock)
+    mocker.patch.dict('freqtrade.exchange._CONF', conf)
+
+    with pytest.raises(
+        OperationalException,
+        match=r'Pair ETH/BTC not compatible with stake_currency: ETH'
+    ):
+        validate_pairs(default_conf['exchange']['pair_whitelist'])
 
 def test_buy_dry_run(default_conf, mocker):
     default_conf['dry_run'] = True
@@ -93,7 +113,6 @@ def test_buy_dry_run(default_conf, mocker):
     assert 'id' in order
     assert 'dry_run_buy_' in order['id']
 
-
 def test_buy_prod(default_conf, mocker):
     api_mock = MagicMock()
     order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6))
@@ -265,6 +284,7 @@ def test_get_ticker(default_conf, mocker):
 
     # retrieve original ticker
     ticker = get_ticker(pair='ETH/BTC')
+
     assert ticker['bid'] == 0.00001098
     assert ticker['ask'] == 0.00001099
 
@@ -281,6 +301,7 @@ def test_get_ticker(default_conf, mocker):
     # if not caching the result we should get the same ticker
     # if not fetching a new result we should get the cached ticker
     ticker = get_ticker(pair='ETH/BTC')
+
     assert ticker['bid'] == 0.5
     assert ticker['ask'] == 1
 

freqtrade/tests/test_configuration.py

@@ -13,6 +13,7 @@ from jsonschema import ValidationError
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.tests.conftest import log_has
+from freqtrade import OperationalException
 
 
 def test_configuration_object() -> None:
@@ -28,7 +29,7 @@ def test_configuration_object() -> None:
     assert hasattr(Configuration, 'get_config')
 
 
-def test_load_config_invalid_pair(default_conf, mocker) -> None:
+def test_load_config_invalid_pair(default_conf) -> None:
     """
     Test the configuration validator with an invalid PAIR format
     """
@@ -40,7 +41,7 @@ def test_load_config_invalid_pair(default_conf, mocker) -> None:
         configuration._validate_config(conf)
 
 
-def test_load_config_missing_attributes(default_conf, mocker) -> None:
+def test_load_config_missing_attributes(default_conf) -> None:
     """
     Test the configuration validator with a missing attribute
     """
@@ -314,3 +315,29 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
     assert 'spaces' in config
     assert config['spaces'] == ['all']
     assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples)
+
+
+def test_check_exchange(default_conf) -> None:
+    """
+    Test the configuration validator with a missing attribute
+    """
+    conf = deepcopy(default_conf)
+    configuration = Configuration([])
+
+    # Test a valid exchange
+    conf.get('exchange').update({'name': 'BITTREX'})
+    assert configuration.check_exchange(conf)
+
+    # Test a valid exchange
+    conf.get('exchange').update({'name': 'binance'})
+    assert configuration.check_exchange(conf)
+
+    # Test a invalid exchange
+    conf.get('exchange').update({'name': 'unknown_exchange'})
+    configuration.config = conf
+
+    with pytest.raises(
+        OperationalException,
+        match=r'.*Exchange "unknown_exchange" not supported.*'
+    ):
+        configuration.check_exchange(conf)

freqtrade/tests/test_misc.py

@@ -71,3 +71,8 @@ def test_file_dump_json(mocker) -> None:
     file_dump_json('somefile', [1, 2, 3])
     assert file_open.call_count == 1
     assert json_dump.call_count == 1
+    file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock())
+    json_dump = mocker.patch('json.dump', MagicMock())
+    file_dump_json('somefile', [1, 2, 3], True)
+    assert file_open.call_count == 1
+    assert json_dump.call_count == 1

scripts/convert_backtestdata.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""
+Script to display when the bot will buy a specific pair
+
+Mandatory Cli parameters:
+-p / --pair: pair to examine
+
+Optional Cli parameters
+-d / --datadir: path to pair backtest data
+--timerange: specify what timerange of data to use.
+-l / --live: Live, to download the latest ticker for the pair
+"""
+
+import sys
+from argparse import Namespace
+from os import path
+import glob
+import json
+import re
+from typing import List, Dict
+import gzip
+
+from freqtrade.arguments import Arguments
+from freqtrade import misc
+from freqtrade.logger import Logger
+from pandas import DataFrame
+
+import dateutil.parser
+
+logger = Logger(name="freqtrade").get_logger()
+
+
+def load_old_file(filename) -> (List[Dict], bool):
+    if not path.isfile(filename):
+        logger.warning("filename %s does not exist", filename)
+        return (None, False)
+    logger.debug('Loading ticker data from file %s', filename)
+
+    pairdata = None
+
+    if filename.endswith('.gz'):
+        logger.debug('Loading ticker data from file %s', filename)
+        is_zip = True
+        with gzip.open(filename) as tickerdata:
+            pairdata = json.load(tickerdata)
+    else:
+        is_zip = False
+        with open(filename) as tickerdata:
+            pairdata = json.load(tickerdata)
+    return (pairdata, is_zip)
+
+
+def parse_old_backtest_data(ticker) -> DataFrame:
+    """
+    Reads old backtest data
+    Format: "O": 8.794e-05,
+            "H": 8.948e-05,
+            "L": 8.794e-05,
+            "C": 8.88e-05,
+            "V": 991.09056638,
+            "T": "2017-11-26T08:50:00",
+            "BV": 0.0877869
+    """
+
+    columns = {'C': 'close', 'V': 'volume', 'O': 'open',
+               'H': 'high', 'L': 'low', 'T': 'date'}
+
+    frame = DataFrame(ticker) \
+        .rename(columns=columns)
+    if 'BV' in frame:
+        frame.drop('BV', 1, inplace=True)
+    if not 'date' in frame:
+        logger.warning("Date not in frame - probably not a Ticker file")
+        return None
+    frame.sort_values('date', inplace=True)
+    return frame
+
+
+def convert_dataframe(frame: DataFrame):
+    """Convert dataframe to new format"""
+    # reorder columns:
+    cols = ['date', 'open', 'high', 'low', 'close', 'volume']
+    frame = frame[cols]
+
+    frame['date'] = frame['date'].apply(
+        lambda d: int(dateutil.parser.parse(d).timestamp()) * 1000)
+    frame['date'] = frame['date'].astype(int)
+    # Convert columns one by one to preserve type.
+    by_column = [frame[x].values.tolist() for x in frame.columns]
+    return list(list(x) for x in zip(*by_column))
+
+
+def convert_file(filename: str, filename_new: str) -> None:
+    """Converts a file from old format to ccxt format"""
+    (pairdata, is_zip) = load_old_file(filename)
+    if pairdata and type(pairdata) is list:
+        if type(pairdata[0]) is list:
+            logger.error("pairdata for %s already in new format", filename)
+            return
+
+    frame = parse_old_backtest_data(pairdata)
+    # Convert frame to new format
+    if frame is not None:
+        frame1 = convert_dataframe(frame)
+        misc.file_dump_json(filename_new, frame1, is_zip)
+
+
+def convert_main(args: Namespace) -> None:
+    """
+    converts a folder given in --datadir from old to new format to support ccxt
+    """
+
+    workdir = path.join(args.datadir, "")
+    logger.info("Workdir: %s", workdir)
+
+    for filename in glob.glob(workdir + "*.json"):
+        # swap currency names
+        ret = re.search(r'[A-Z_]{7,}', path.basename(filename))
+        if args.norename:
+            filename_new = filename
+        else:
+            if not ret:
+                logger.warning("file %s could not be converted, could not extract currencies",
+                               filename)
+                continue
+            pair = ret.group(0)
+            currencies = pair.split("_")
+            if len(currencies) != 2:
+                logger.warning("file %s could not be converted, could not extract currencies",
+                               filename)
+                continue
+
+            ret = re.search(r'\d+(?=\.json)', path.basename(filename))
+            if not ret:
+                logger.warning("file %s could not be converted, interval not found", filename)
+                continue
+            interval = ret.group(0)
+
+            filename_new = path.join(path.dirname(filename),
+                                     "{}_{}-{}.json".format(currencies[1],
+                                                            currencies[0], interval))
+        logger.debug("Converting and renaming %s to %s", filename, filename_new)
+        convert_file(filename, filename_new)
+
+
+def convert_parse_args(args: List[str]) -> Namespace:
+    """
+    Parse args passed to the script
+    :param args: Cli arguments
+    :return: args: Array with all arguments
+    """
+    arguments = Arguments(args, 'Convert datafiles')
+    arguments.parser.add_argument(
+        '-d', '--datadir',
+        help='path to backtest data (default: %(default)s',
+        dest='datadir',
+        default=path.join('freqtrade', 'tests', 'testdata'),
+        type=str,
+        metavar='PATH',
+    )
+    arguments.parser.add_argument(
+        '-n', '--norename',
+        help='don''t rename files from BTC_<PAIR> to <PAIR>_BTC - '
+             'Note that not renaming will overwrite source files',
+        dest='norename',
+        default=False,
+        action='store_true'
+    )
+
+    return arguments.parse_args()
+
+
+def main(sysargv: List[str]) -> None:
+    """
+    This function will initiate the bot and start the trading loop.
+    :return: None
+    """
+    logger.info('Starting Dataframe conversation')
+    convert_main(convert_parse_args(sysargv))
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])