From 89e3729955f8671c21f72d950860037dc7058cdd Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Feb 2018 22:42:03 -0800 Subject: [PATCH] Add a Configuration class that generate the Bot config from Arguments --- freqtrade/configuration.py | 102 ++++++++++ freqtrade/tests/conftest.py | 12 +- freqtrade/tests/conftest_old.py | 281 -------------------------- freqtrade/tests/test_configuration.py | 157 ++++++++++++++ 4 files changed, 266 insertions(+), 286 deletions(-) create mode 100644 freqtrade/configuration.py delete mode 100644 freqtrade/tests/conftest_old.py create mode 100644 freqtrade/tests/test_configuration.py diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py new file mode 100644 index 000000000..72925e8ce --- /dev/null +++ b/freqtrade/configuration.py @@ -0,0 +1,102 @@ +""" +This module contains the configuration class +""" + +import json + +from typing import Dict, List, Any +from jsonschema import Draft4Validator, validate +from jsonschema.exceptions import ValidationError, best_match + +from freqtrade.constants import Constants +from freqtrade.logger import Logger + + +class Configuration(object): + """ + Class to read and init the bot configuration + Reuse this class for the bot, backtesting, hyperopt and every script that required configuration + """ + def __init__(self, args: List[str]) -> None: + self.args = args + self.logger = Logger(name=__name__).get_logger() + self.config = self._load_config() + + def _load_config(self) -> Dict[str, Any]: + """ + Extract information for sys.argv and load the bot configuration + :return: Configuration dictionary + """ + config = self._load_config_file(self.args.config) + + # Add the strategy file to use + config.update({'strategy': self.args.strategy}) + + # Add dynamic_whitelist if found + if self.args.dynamic_whitelist: + config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) + + # Add dry_run_db if found and the bot in dry run + if self.args.dry_run_db and config.get('dry_run', False): + config.update({'dry_run_db': True}) + + return config + + def _load_config_file(self, path: str) -> Dict[str, Any]: + """ + Loads a config file from the given path + :param path: path as str + :return: configuration as dictionary + """ + with open(path) as file: + conf = json.load(file) + + if 'internals' not in conf: + conf['internals'] = {} + self.logger.info('Validating configuration ...') + + return self._validate_config(conf) + + def _validate_config(self, conf: Dict[str, Any]) -> Dict[str, Any]: + """ + Validate the configuration follow the Config Schema + :param conf: Config in JSON format + :return: Returns the config if valid, otherwise throw an exception + """ + try: + validate(conf, Constants.CONF_SCHEMA) + return conf + except ValidationError as exception: + self.logger.fatal( + 'Invalid configuration. See config.json.example. Reason: %s', + exception + ) + raise ValidationError( + best_match(Draft4Validator(Constants.CONF_SCHEMA).iter_errors(conf)).message + ) + + def show_info(self) -> None: + """ + Display info message to user depending of the configuration of the bot + :return: None + """ + if self.config.get('dynamic_whitelist', False): + self.logger.info( + 'Using dynamically generated whitelist. (--dynamic-whitelist detected)' + ) + + if self.config.get('dry_run_db', False): + if self.config.get('dry_run', False): + self.logger.info( + 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". ' + '(--dry_run_db detected)' + ) + else: + self.logger.info('Dry run is disabled. (--dry_run_db ignored)') + + def get_config(self) -> Dict[str, Any]: + """ + Return the config. Use this method to get the bot config + :return: Dict: Bot config + """ + return self.config diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 2b1d14268..e2967b845 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,16 +2,18 @@ from datetime import datetime from unittest.mock import MagicMock from functools import reduce - +import logging import json import arrow import pytest from jsonschema import validate from telegram import Chat, Message, Update -from freqtrade.analyze import parse_ticker_dataframe + +from freqtrade.analyze import Analyze +from freqtrade.constants import Constants from freqtrade.strategy.strategy import Strategy -from freqtrade.misc import CONF_SCHEMA +logging.getLogger('').setLevel(logging.INFO) def log_has(line, logs): @@ -63,7 +65,7 @@ def default_conf(): }, "initial_state": "running" } - validate(configuration, CONF_SCHEMA) + validate(configuration, Constants.CONF_SCHEMA) return configuration @@ -264,7 +266,7 @@ def ticker_history_without_bv(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) + return Analyze.parse_ticker_dataframe(json.load(data_file)) @pytest.fixture diff --git a/freqtrade/tests/conftest_old.py b/freqtrade/tests/conftest_old.py deleted file mode 100644 index 2b1d14268..000000000 --- a/freqtrade/tests/conftest_old.py +++ /dev/null @@ -1,281 +0,0 @@ -# pragma pylint: disable=missing-docstring -from datetime import datetime -from unittest.mock import MagicMock -from functools import reduce - -import json -import arrow -import pytest -from jsonschema import validate -from telegram import Chat, Message, Update -from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.strategy.strategy import Strategy - -from freqtrade.misc import CONF_SCHEMA - - -def log_has(line, logs): - # caplog mocker returns log as a tuple: ('freqtrade.analyze', logging.WARNING, 'foobar') - # and we want to match line against foobar in the tuple - return reduce(lambda a, b: a or b, - filter(lambda x: x[2] == line, logs), - False) - - -@pytest.fixture(scope="module") -def default_conf(): - """ Returns validated configuration suitable for most tests """ - configuration = { - "max_open_trades": 1, - "stake_currency": "BTC", - "stake_amount": 0.001, - "fiat_display_currency": "USD", - "ticker_interval": 5, - "dry_run": True, - "minimal_roi": { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 - }, - "stoploss": -0.10, - "unfilledtimeout": 600, - "bid_strategy": { - "ask_last_balance": 0.0 - }, - "exchange": { - "name": "bittrex", - "enabled": True, - "key": "key", - "secret": "secret", - "pair_whitelist": [ - "BTC_ETH", - "BTC_TKN", - "BTC_TRST", - "BTC_SWT", - "BTC_BCC" - ] - }, - "telegram": { - "enabled": True, - "token": "token", - "chat_id": "0" - }, - "initial_state": "running" - } - validate(configuration, CONF_SCHEMA) - return configuration - - -@pytest.fixture -def update(): - _update = Update(0) - _update.message = Message(0, 0, datetime.utcnow(), Chat(0, 0)) - return _update - - -@pytest.fixture -def ticker(): - return MagicMock(return_value={ - 'bid': 0.00001098, - 'ask': 0.00001099, - 'last': 0.00001098, - }) - - -@pytest.fixture -def ticker_sell_up(): - return MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172, - }) - - -@pytest.fixture -def ticker_sell_down(): - return MagicMock(return_value={ - 'bid': 0.00001044, - 'ask': 0.00001043, - 'last': 0.00001044, - }) - - -@pytest.fixture -def health(): - return MagicMock(return_value=[{ - 'Currency': 'BTC', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'ETH', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'TRST', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'SWT', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'BCC', - 'IsActive': False, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }]) - - -@pytest.fixture -def limit_buy_order(): - return { - 'id': 'mocked_limit_buy', - 'type': 'LIMIT_BUY', - 'pair': 'mocked', - 'opened': str(arrow.utcnow().datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 0.0, - 'closed': str(arrow.utcnow().datetime), - } - - -@pytest.fixture -def limit_buy_order_old(): - return { - 'id': 'mocked_limit_buy_old', - 'type': 'LIMIT_BUY', - 'pair': 'BTC_ETH', - 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 90.99181073, - } - - -@pytest.fixture -def limit_sell_order_old(): - return { - 'id': 'mocked_limit_sell_old', - 'type': 'LIMIT_SELL', - 'pair': 'BTC_ETH', - 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 90.99181073, - } - - -@pytest.fixture -def limit_buy_order_old_partial(): - return { - 'id': 'mocked_limit_buy_old_partial', - 'type': 'LIMIT_BUY', - 'pair': 'BTC_ETH', - 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 67.99181073, - } - - -@pytest.fixture -def limit_sell_order(): - return { - 'id': 'mocked_limit_sell', - 'type': 'LIMIT_SELL', - 'pair': 'mocked', - 'opened': str(arrow.utcnow().datetime), - 'rate': 0.00001173, - 'amount': 90.99181073, - 'remaining': 0.0, - 'closed': str(arrow.utcnow().datetime), - } - - -@pytest.fixture -def ticker_history(): - return [ - { - "O": 8.794e-05, - "H": 8.948e-05, - "L": 8.794e-05, - "C": 8.88e-05, - "V": 991.09056638, - "T": "2017-11-26T08:50:00", - "BV": 0.0877869 - }, - { - "O": 8.88e-05, - "H": 8.942e-05, - "L": 8.88e-05, - "C": 8.893e-05, - "V": 658.77935965, - "T": "2017-11-26T08:55:00", - "BV": 0.05874751 - }, - { - "O": 8.891e-05, - "H": 8.893e-05, - "L": 8.875e-05, - "C": 8.877e-05, - "V": 7920.73570705, - "T": "2017-11-26T09:00:00", - "BV": 0.7039405 - } - ] - - -@pytest.fixture -def ticker_history_without_bv(): - return [ - { - "O": 8.794e-05, - "H": 8.948e-05, - "L": 8.794e-05, - "C": 8.88e-05, - "V": 991.09056638, - "T": "2017-11-26T08:50:00" - }, - { - "O": 8.88e-05, - "H": 8.942e-05, - "L": 8.88e-05, - "C": 8.893e-05, - "V": 658.77935965, - "T": "2017-11-26T08:55:00" - }, - { - "O": 8.891e-05, - "H": 8.893e-05, - "L": 8.875e-05, - "C": 8.877e-05, - "V": 7920.73570705, - "T": "2017-11-26T09:00:00" - } - ] - - -@pytest.fixture -def result(): - with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) - - -@pytest.fixture -def default_strategy(): - strategy = Strategy() - strategy.init({'strategy': 'default_strategy'}) - return strategy - - -# FIX: -# Create an fixture/function -# that inserts a trade of some type and open-status -# return the open-order-id -# See tests in rpc/main that could use this diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py new file mode 100644 index 000000000..c9def6a9e --- /dev/null +++ b/freqtrade/tests/test_configuration.py @@ -0,0 +1,157 @@ +# pragma pylint: disable=protected-access, invalid-name, missing-docstring + +""" +Unit test file for configuration.py +""" +import json + +from copy import deepcopy +from unittest.mock import MagicMock +import pytest +from jsonschema import ValidationError + +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +import freqtrade.tests.conftest as tt # test tools + + +def test_configuration_object() -> None: + """ + Test the Constants object has the mandatory Constants + :return: None + """ + assert hasattr(Configuration, '_load_config') + assert hasattr(Configuration, '_load_config_file') + assert hasattr(Configuration, '_validate_config') + assert hasattr(Configuration, 'show_info') + assert hasattr(Configuration, 'get_config') + + +def test_load_config_invalid_pair(default_conf, mocker) -> None: + """ + Test the configuration validator with an invalid PAIR format + :param default_conf: Configuration already read from a file (JSON format) + :return: None + """ + mocker.patch.multiple( + 'freqtrade.configuration.Configuration', + _load_config=MagicMock(return_value=[]) + ) + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'].append('BTC-ETH') + + with pytest.raises(ValidationError, match=r'.*does not match.*'): + configuration = Configuration([]) + configuration._validate_config(conf) + + +def test_load_config_missing_attributes(default_conf, mocker) -> None: + """ + Test the configuration validator with a missing attribute + :param default_conf: Configuration already read from a file (JSON format) + :return: None + """ + mocker.patch.multiple( + 'freqtrade.configuration.Configuration', + _load_config=MagicMock(return_value=[]) + ) + conf = deepcopy(default_conf) + conf.pop('exchange') + + with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): + configuration = Configuration([]) + configuration._validate_config(conf) + + +def test_load_config_file(default_conf, mocker, caplog) -> None: + """ + Test _load_config_file() method + :return: + """ + mocker.patch.multiple( + 'freqtrade.configuration.Configuration', + _load_config=MagicMock(return_value=[]) + ) + file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + configuration = Configuration([]) + validated_conf = configuration._load_config_file('somefile') + assert file_mock.call_count == 1 + assert validated_conf.items() >= default_conf.items() + assert 'internals' in validated_conf + assert tt.log_has('Validating configuration ...', caplog.record_tuples) + + +def test_load_config(default_conf, mocker) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration._load_config() + + assert 'strategy' in validated_conf + assert validated_conf['strategy'] == 'default_strategy' + assert 'dynamic_whitelist' not in validated_conf + assert 'dry_run_db' not in validated_conf + + +def test_load_config_with_params(default_conf, mocker) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--dynamic-whitelist', '10', + '--strategy', 'test_strategy', + '--dry-run-db' + ] + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration._load_config() + + assert 'dynamic_whitelist' in validated_conf + assert validated_conf['dynamic_whitelist'] == 10 + assert 'strategy' in validated_conf + assert validated_conf['strategy'] == 'test_strategy' + assert 'dry_run_db' in validated_conf + assert validated_conf['dry_run_db'] is True + + +def test_show_info(default_conf, mocker, caplog) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--dynamic-whitelist', '10', + '--strategy', 'test_strategy', + '--dry-run-db' + ] + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + configuration.show_info() + + assert tt.log_has( + 'Using dynamically generated whitelist. (--dynamic-whitelist detected)', + caplog.record_tuples + ) + + assert tt.log_has( + 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". ' + '(--dry_run_db detected)', + caplog.record_tuples + ) + + # Test the Dry run condition + configuration.config.update({'dry_run': False}) + configuration.show_info() + assert tt.log_has( + 'Dry run is disabled. (--dry_run_db ignored)', + caplog.record_tuples + )