From 314ab0a84f6bf643b2376d109864e0e74a4f82f9 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Feb 2018 16:04:26 -0800 Subject: [PATCH 01/56] Add a Constants class that contains Bot constants --- freqtrade/constants.py | 121 +++++++++++++ freqtrade/tests/conftest_old.py | 281 ++++++++++++++++++++++++++++++ freqtrade/tests/test_constants.py | 28 +++ 3 files changed, 430 insertions(+) create mode 100644 freqtrade/constants.py create mode 100644 freqtrade/tests/conftest_old.py create mode 100644 freqtrade/tests/test_constants.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py new file mode 100644 index 000000000..765a4fe3e --- /dev/null +++ b/freqtrade/constants.py @@ -0,0 +1,121 @@ +# pragma pylint: disable=too-few-public-methods + +""" +List bot constants +""" + + +class Constants(object): + """ + Static class that contain all bot constants + """ + DYNAMIC_WHITELIST = 20 # pairs + PROCESS_THROTTLE_SECS = 5 # sec + TICKER_INTERVAL = 5 # min + HYPEROPT_EPOCH = 100 # epochs + RETRY_TIMEOUT = 30 # sec + + # Required json-schema for user specified config + CONF_SCHEMA = { + 'type': 'object', + 'properties': { + 'max_open_trades': {'type': 'integer', 'minimum': 1}, + 'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, + 'stake_amount': {'type': 'number', 'minimum': 0.0005}, + 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', + 'CLP', 'CNY', 'CZK', 'DKK', + 'EUR', 'GBP', 'HKD', 'HUF', + 'IDR', 'ILS', 'INR', 'JPY', + 'KRW', 'MXN', 'MYR', 'NOK', + 'NZD', 'PHP', 'PKR', 'PLN', + 'RUB', 'SEK', 'SGD', 'THB', + 'TRY', 'TWD', 'ZAR', 'USD']}, + 'dry_run': {'type': 'boolean'}, + 'minimal_roi': { + 'type': 'object', + 'patternProperties': { + '^[0-9.]+$': {'type': 'number'} + }, + 'minProperties': 1 + }, + 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, + 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'bid_strategy': { + 'type': 'object', + 'properties': { + 'ask_last_balance': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1, + 'exclusiveMaximum': False + }, + }, + 'required': ['ask_last_balance'] + }, + 'exchange': {'$ref': '#/definitions/exchange'}, + 'experimental': { + 'type': 'object', + 'properties': { + 'use_sell_signal': {'type': 'boolean'}, + 'sell_profit_only': {'type': 'boolean'} + } + }, + 'telegram': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'token': {'type': 'string'}, + 'chat_id': {'type': 'string'}, + }, + 'required': ['enabled', 'token', 'chat_id'] + }, + 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, + 'internals': { + 'type': 'object', + 'properties': { + 'process_throttle_secs': {'type': 'number'}, + 'interval': {'type': 'integer'} + } + } + }, + 'definitions': { + 'exchange': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'key': {'type': 'string'}, + 'secret': {'type': 'string'}, + 'pair_whitelist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' + }, + 'uniqueItems': True + }, + 'pair_blacklist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' + }, + 'uniqueItems': True + } + }, + 'required': ['name', 'key', 'secret', 'pair_whitelist'] + } + }, + 'anyOf': [ + {'required': ['exchange']} + ], + 'required': [ + 'max_open_trades', + 'stake_currency', + 'stake_amount', + 'fiat_display_currency', + 'dry_run', + 'bid_strategy', + 'telegram' + ] + } diff --git a/freqtrade/tests/conftest_old.py b/freqtrade/tests/conftest_old.py new file mode 100644 index 000000000..2b1d14268 --- /dev/null +++ b/freqtrade/tests/conftest_old.py @@ -0,0 +1,281 @@ +# pragma pylint: disable=missing-docstring +from datetime import datetime +from unittest.mock import MagicMock +from functools import reduce + +import json +import arrow +import pytest +from jsonschema import validate +from telegram import Chat, Message, Update +from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.strategy.strategy import Strategy + +from freqtrade.misc import CONF_SCHEMA + + +def log_has(line, logs): + # caplog mocker returns log as a tuple: ('freqtrade.analyze', logging.WARNING, 'foobar') + # and we want to match line against foobar in the tuple + return reduce(lambda a, b: a or b, + filter(lambda x: x[2] == line, logs), + False) + + +@pytest.fixture(scope="module") +def default_conf(): + """ Returns validated configuration suitable for most tests """ + configuration = { + "max_open_trades": 1, + "stake_currency": "BTC", + "stake_amount": 0.001, + "fiat_display_currency": "USD", + "ticker_interval": 5, + "dry_run": True, + "minimal_roi": { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + }, + "stoploss": -0.10, + "unfilledtimeout": 600, + "bid_strategy": { + "ask_last_balance": 0.0 + }, + "exchange": { + "name": "bittrex", + "enabled": True, + "key": "key", + "secret": "secret", + "pair_whitelist": [ + "BTC_ETH", + "BTC_TKN", + "BTC_TRST", + "BTC_SWT", + "BTC_BCC" + ] + }, + "telegram": { + "enabled": True, + "token": "token", + "chat_id": "0" + }, + "initial_state": "running" + } + validate(configuration, CONF_SCHEMA) + return configuration + + +@pytest.fixture +def update(): + _update = Update(0) + _update.message = Message(0, 0, datetime.utcnow(), Chat(0, 0)) + return _update + + +@pytest.fixture +def ticker(): + return MagicMock(return_value={ + 'bid': 0.00001098, + 'ask': 0.00001099, + 'last': 0.00001098, + }) + + +@pytest.fixture +def ticker_sell_up(): + return MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172, + }) + + +@pytest.fixture +def ticker_sell_down(): + return MagicMock(return_value={ + 'bid': 0.00001044, + 'ask': 0.00001043, + 'last': 0.00001044, + }) + + +@pytest.fixture +def health(): + return MagicMock(return_value=[{ + 'Currency': 'BTC', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'ETH', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'TRST', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'SWT', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'BCC', + 'IsActive': False, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }]) + + +@pytest.fixture +def limit_buy_order(): + return { + 'id': 'mocked_limit_buy', + 'type': 'LIMIT_BUY', + 'pair': 'mocked', + 'opened': str(arrow.utcnow().datetime), + 'rate': 0.00001099, + 'amount': 90.99181073, + 'remaining': 0.0, + 'closed': str(arrow.utcnow().datetime), + } + + +@pytest.fixture +def limit_buy_order_old(): + return { + 'id': 'mocked_limit_buy_old', + 'type': 'LIMIT_BUY', + 'pair': 'BTC_ETH', + 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), + 'rate': 0.00001099, + 'amount': 90.99181073, + 'remaining': 90.99181073, + } + + +@pytest.fixture +def limit_sell_order_old(): + return { + 'id': 'mocked_limit_sell_old', + 'type': 'LIMIT_SELL', + 'pair': 'BTC_ETH', + 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), + 'rate': 0.00001099, + 'amount': 90.99181073, + 'remaining': 90.99181073, + } + + +@pytest.fixture +def limit_buy_order_old_partial(): + return { + 'id': 'mocked_limit_buy_old_partial', + 'type': 'LIMIT_BUY', + 'pair': 'BTC_ETH', + 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), + 'rate': 0.00001099, + 'amount': 90.99181073, + 'remaining': 67.99181073, + } + + +@pytest.fixture +def limit_sell_order(): + return { + 'id': 'mocked_limit_sell', + 'type': 'LIMIT_SELL', + 'pair': 'mocked', + 'opened': str(arrow.utcnow().datetime), + 'rate': 0.00001173, + 'amount': 90.99181073, + 'remaining': 0.0, + 'closed': str(arrow.utcnow().datetime), + } + + +@pytest.fixture +def ticker_history(): + return [ + { + "O": 8.794e-05, + "H": 8.948e-05, + "L": 8.794e-05, + "C": 8.88e-05, + "V": 991.09056638, + "T": "2017-11-26T08:50:00", + "BV": 0.0877869 + }, + { + "O": 8.88e-05, + "H": 8.942e-05, + "L": 8.88e-05, + "C": 8.893e-05, + "V": 658.77935965, + "T": "2017-11-26T08:55:00", + "BV": 0.05874751 + }, + { + "O": 8.891e-05, + "H": 8.893e-05, + "L": 8.875e-05, + "C": 8.877e-05, + "V": 7920.73570705, + "T": "2017-11-26T09:00:00", + "BV": 0.7039405 + } + ] + + +@pytest.fixture +def ticker_history_without_bv(): + return [ + { + "O": 8.794e-05, + "H": 8.948e-05, + "L": 8.794e-05, + "C": 8.88e-05, + "V": 991.09056638, + "T": "2017-11-26T08:50:00" + }, + { + "O": 8.88e-05, + "H": 8.942e-05, + "L": 8.88e-05, + "C": 8.893e-05, + "V": 658.77935965, + "T": "2017-11-26T08:55:00" + }, + { + "O": 8.891e-05, + "H": 8.893e-05, + "L": 8.875e-05, + "C": 8.877e-05, + "V": 7920.73570705, + "T": "2017-11-26T09:00:00" + } + ] + + +@pytest.fixture +def result(): + with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: + return parse_ticker_dataframe(json.load(data_file)) + + +@pytest.fixture +def default_strategy(): + strategy = Strategy() + strategy.init({'strategy': 'default_strategy'}) + return strategy + + +# FIX: +# Create an fixture/function +# that inserts a trade of some type and open-status +# return the open-order-id +# See tests in rpc/main that could use this diff --git a/freqtrade/tests/test_constants.py b/freqtrade/tests/test_constants.py new file mode 100644 index 000000000..e50fbb880 --- /dev/null +++ b/freqtrade/tests/test_constants.py @@ -0,0 +1,28 @@ +""" +Unit test file for constants.py +""" + +from freqtrade.constants import Constants + + +def test_constant_object() -> None: + """ + Test the Constants object has the mandatory Constants + :return: None + """ + constant = Constants() + assert hasattr(constant, 'CONF_SCHEMA') + assert hasattr(constant, 'DYNAMIC_WHITELIST') + assert hasattr(constant, 'PROCESS_THROTTLE_SECS') + assert hasattr(constant, 'TICKER_INTERVAL') + assert hasattr(constant, 'HYPEROPT_EPOCH') + assert hasattr(constant, 'RETRY_TIMEOUT') + + +def test_conf_schema() -> None: + """ + Test the CONF_SCHEMA is from the right type + :return: + """ + constant = Constants() + assert isinstance(constant.CONF_SCHEMA, dict) From cf753d5c404fcdd631acd6e0cf5d8bdc7301b10a Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Feb 2018 16:10:58 -0800 Subject: [PATCH 02/56] Add a Enum class State that contains Bot running states --- freqtrade/state.py | 14 ++++++++++++++ freqtrade/tests/test_state.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 freqtrade/state.py create mode 100644 freqtrade/tests/test_state.py diff --git a/freqtrade/state.py b/freqtrade/state.py new file mode 100644 index 000000000..aaa5b4765 --- /dev/null +++ b/freqtrade/state.py @@ -0,0 +1,14 @@ +# pragma pylint: disable=too-few-public-methods + +""" +Bot state constant +""" +import enum + + +class State(enum.Enum): + """ + Bot running states + """ + RUNNING = 0 + STOPPED = 1 diff --git a/freqtrade/tests/test_state.py b/freqtrade/tests/test_state.py new file mode 100644 index 000000000..51fa06cc2 --- /dev/null +++ b/freqtrade/tests/test_state.py @@ -0,0 +1,14 @@ +""" +Unit test file for constants.py +""" + +from freqtrade.state import State + + +def test_state_object() -> None: + """ + Test the State object has the mandatory states + :return: None + """ + assert hasattr(State, 'RUNNING') + assert hasattr(State, 'STOPPED') From 3b9e828fa44d690d98649fb4131b1528b4463c8a Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Feb 2018 16:19:16 -0800 Subject: [PATCH 03/56] Add a class Logger to manage the logging messages This class will evolve later to support color logging. For now it is used to not repeat the logging configuration everywhere. --- freqtrade/logger.py | 40 +++++++++++++++++++++++++ freqtrade/tests/test_logger.py | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 freqtrade/logger.py create mode 100644 freqtrade/tests/test_logger.py diff --git a/freqtrade/logger.py b/freqtrade/logger.py new file mode 100644 index 000000000..4526ac11c --- /dev/null +++ b/freqtrade/logger.py @@ -0,0 +1,40 @@ +# pragma pylint: disable=too-few-public-methods + +""" +This module contains the class for logger and logging messages +""" + +import logging + + +class Logger(object): + """ + Logging class + """ + def __init__(self, name='', level=logging.INFO) -> None: + """ + Init the logger class + :param name: Name of the Logger scope + :param level: Logger level that should be used + :return: None + """ + self.name = name + self.level = level + self._init_logger() + + def _init_logger(self) -> logging: + """ + Setup the bot logger configuration + :return: logging object + """ + logging.basicConfig( + level=self.level, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + + def get_logger(self) -> logging.RootLogger: + """ + Return the logger instance to use for sending message + :return: the logger instance + """ + return logging.getLogger(self.name) diff --git a/freqtrade/tests/test_logger.py b/freqtrade/tests/test_logger.py new file mode 100644 index 000000000..e492ce097 --- /dev/null +++ b/freqtrade/tests/test_logger.py @@ -0,0 +1,53 @@ +""" +Unit test file for logger.py +""" + +import logging +from freqtrade.logger import Logger + + +def test_logger_object() -> None: + """ + Test the Constants object has the mandatory Constants + :return: None + """ + logger = Logger() + assert logger.name == '' + assert logger.level == 20 + assert logger.level is logging.INFO + assert hasattr(logger, 'get_logger') + + logger = Logger(name='Foo', level=logging.WARNING) + assert logger.name == 'Foo' + assert logger.name != '' + assert logger.level == 30 + assert logger.level is logging.WARNING + + +def test_get_logger() -> None: + """ + Test logger.get_logger() + :return: None + """ + logger = Logger(name='Foo', level=logging.WARNING) + get_logger = logger.get_logger() + assert get_logger is not None + assert hasattr(get_logger, 'debug') + assert hasattr(get_logger, 'info') + assert hasattr(get_logger, 'warning') + assert hasattr(get_logger, 'critical') + assert hasattr(get_logger, 'exception') + + +def test_sending_msg(caplog) -> None: + """ + Test send a logging message + :return: None + """ + logger = Logger(name='FooBar', level=logging.WARNING).get_logger() + + logger.info('I am an INFO message') + assert('FooBar', logging.INFO, 'I am an INFO message') not in caplog.record_tuples + + logger.warning('I am an WARNING message') + assert ('FooBar', logging.WARNING, 'I am an WARNING message') in caplog.record_tuples From 89e3729955f8671c21f72d950860037dc7058cdd Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Feb 2018 22:42:03 -0800 Subject: [PATCH 04/56] Add a Configuration class that generate the Bot config from Arguments --- freqtrade/configuration.py | 102 ++++++++++ freqtrade/tests/conftest.py | 12 +- freqtrade/tests/conftest_old.py | 281 -------------------------- freqtrade/tests/test_configuration.py | 157 ++++++++++++++ 4 files changed, 266 insertions(+), 286 deletions(-) create mode 100644 freqtrade/configuration.py delete mode 100644 freqtrade/tests/conftest_old.py create mode 100644 freqtrade/tests/test_configuration.py diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py new file mode 100644 index 000000000..72925e8ce --- /dev/null +++ b/freqtrade/configuration.py @@ -0,0 +1,102 @@ +""" +This module contains the configuration class +""" + +import json + +from typing import Dict, List, Any +from jsonschema import Draft4Validator, validate +from jsonschema.exceptions import ValidationError, best_match + +from freqtrade.constants import Constants +from freqtrade.logger import Logger + + +class Configuration(object): + """ + Class to read and init the bot configuration + Reuse this class for the bot, backtesting, hyperopt and every script that required configuration + """ + def __init__(self, args: List[str]) -> None: + self.args = args + self.logger = Logger(name=__name__).get_logger() + self.config = self._load_config() + + def _load_config(self) -> Dict[str, Any]: + """ + Extract information for sys.argv and load the bot configuration + :return: Configuration dictionary + """ + config = self._load_config_file(self.args.config) + + # Add the strategy file to use + config.update({'strategy': self.args.strategy}) + + # Add dynamic_whitelist if found + if self.args.dynamic_whitelist: + config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) + + # Add dry_run_db if found and the bot in dry run + if self.args.dry_run_db and config.get('dry_run', False): + config.update({'dry_run_db': True}) + + return config + + def _load_config_file(self, path: str) -> Dict[str, Any]: + """ + Loads a config file from the given path + :param path: path as str + :return: configuration as dictionary + """ + with open(path) as file: + conf = json.load(file) + + if 'internals' not in conf: + conf['internals'] = {} + self.logger.info('Validating configuration ...') + + return self._validate_config(conf) + + def _validate_config(self, conf: Dict[str, Any]) -> Dict[str, Any]: + """ + Validate the configuration follow the Config Schema + :param conf: Config in JSON format + :return: Returns the config if valid, otherwise throw an exception + """ + try: + validate(conf, Constants.CONF_SCHEMA) + return conf + except ValidationError as exception: + self.logger.fatal( + 'Invalid configuration. See config.json.example. Reason: %s', + exception + ) + raise ValidationError( + best_match(Draft4Validator(Constants.CONF_SCHEMA).iter_errors(conf)).message + ) + + def show_info(self) -> None: + """ + Display info message to user depending of the configuration of the bot + :return: None + """ + if self.config.get('dynamic_whitelist', False): + self.logger.info( + 'Using dynamically generated whitelist. (--dynamic-whitelist detected)' + ) + + if self.config.get('dry_run_db', False): + if self.config.get('dry_run', False): + self.logger.info( + 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". ' + '(--dry_run_db detected)' + ) + else: + self.logger.info('Dry run is disabled. (--dry_run_db ignored)') + + def get_config(self) -> Dict[str, Any]: + """ + Return the config. Use this method to get the bot config + :return: Dict: Bot config + """ + return self.config diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 2b1d14268..e2967b845 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,16 +2,18 @@ from datetime import datetime from unittest.mock import MagicMock from functools import reduce - +import logging import json import arrow import pytest from jsonschema import validate from telegram import Chat, Message, Update -from freqtrade.analyze import parse_ticker_dataframe + +from freqtrade.analyze import Analyze +from freqtrade.constants import Constants from freqtrade.strategy.strategy import Strategy -from freqtrade.misc import CONF_SCHEMA +logging.getLogger('').setLevel(logging.INFO) def log_has(line, logs): @@ -63,7 +65,7 @@ def default_conf(): }, "initial_state": "running" } - validate(configuration, CONF_SCHEMA) + validate(configuration, Constants.CONF_SCHEMA) return configuration @@ -264,7 +266,7 @@ def ticker_history_without_bv(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) + return Analyze.parse_ticker_dataframe(json.load(data_file)) @pytest.fixture diff --git a/freqtrade/tests/conftest_old.py b/freqtrade/tests/conftest_old.py deleted file mode 100644 index 2b1d14268..000000000 --- a/freqtrade/tests/conftest_old.py +++ /dev/null @@ -1,281 +0,0 @@ -# pragma pylint: disable=missing-docstring -from datetime import datetime -from unittest.mock import MagicMock -from functools import reduce - -import json -import arrow -import pytest -from jsonschema import validate -from telegram import Chat, Message, Update -from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.strategy.strategy import Strategy - -from freqtrade.misc import CONF_SCHEMA - - -def log_has(line, logs): - # caplog mocker returns log as a tuple: ('freqtrade.analyze', logging.WARNING, 'foobar') - # and we want to match line against foobar in the tuple - return reduce(lambda a, b: a or b, - filter(lambda x: x[2] == line, logs), - False) - - -@pytest.fixture(scope="module") -def default_conf(): - """ Returns validated configuration suitable for most tests """ - configuration = { - "max_open_trades": 1, - "stake_currency": "BTC", - "stake_amount": 0.001, - "fiat_display_currency": "USD", - "ticker_interval": 5, - "dry_run": True, - "minimal_roi": { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 - }, - "stoploss": -0.10, - "unfilledtimeout": 600, - "bid_strategy": { - "ask_last_balance": 0.0 - }, - "exchange": { - "name": "bittrex", - "enabled": True, - "key": "key", - "secret": "secret", - "pair_whitelist": [ - "BTC_ETH", - "BTC_TKN", - "BTC_TRST", - "BTC_SWT", - "BTC_BCC" - ] - }, - "telegram": { - "enabled": True, - "token": "token", - "chat_id": "0" - }, - "initial_state": "running" - } - validate(configuration, CONF_SCHEMA) - return configuration - - -@pytest.fixture -def update(): - _update = Update(0) - _update.message = Message(0, 0, datetime.utcnow(), Chat(0, 0)) - return _update - - -@pytest.fixture -def ticker(): - return MagicMock(return_value={ - 'bid': 0.00001098, - 'ask': 0.00001099, - 'last': 0.00001098, - }) - - -@pytest.fixture -def ticker_sell_up(): - return MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172, - }) - - -@pytest.fixture -def ticker_sell_down(): - return MagicMock(return_value={ - 'bid': 0.00001044, - 'ask': 0.00001043, - 'last': 0.00001044, - }) - - -@pytest.fixture -def health(): - return MagicMock(return_value=[{ - 'Currency': 'BTC', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'ETH', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'TRST', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'SWT', - 'IsActive': True, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }, { - 'Currency': 'BCC', - 'IsActive': False, - 'LastChecked': '2017-11-13T20:15:00.00', - 'Notice': None - }]) - - -@pytest.fixture -def limit_buy_order(): - return { - 'id': 'mocked_limit_buy', - 'type': 'LIMIT_BUY', - 'pair': 'mocked', - 'opened': str(arrow.utcnow().datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 0.0, - 'closed': str(arrow.utcnow().datetime), - } - - -@pytest.fixture -def limit_buy_order_old(): - return { - 'id': 'mocked_limit_buy_old', - 'type': 'LIMIT_BUY', - 'pair': 'BTC_ETH', - 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 90.99181073, - } - - -@pytest.fixture -def limit_sell_order_old(): - return { - 'id': 'mocked_limit_sell_old', - 'type': 'LIMIT_SELL', - 'pair': 'BTC_ETH', - 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 90.99181073, - } - - -@pytest.fixture -def limit_buy_order_old_partial(): - return { - 'id': 'mocked_limit_buy_old_partial', - 'type': 'LIMIT_BUY', - 'pair': 'BTC_ETH', - 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), - 'rate': 0.00001099, - 'amount': 90.99181073, - 'remaining': 67.99181073, - } - - -@pytest.fixture -def limit_sell_order(): - return { - 'id': 'mocked_limit_sell', - 'type': 'LIMIT_SELL', - 'pair': 'mocked', - 'opened': str(arrow.utcnow().datetime), - 'rate': 0.00001173, - 'amount': 90.99181073, - 'remaining': 0.0, - 'closed': str(arrow.utcnow().datetime), - } - - -@pytest.fixture -def ticker_history(): - return [ - { - "O": 8.794e-05, - "H": 8.948e-05, - "L": 8.794e-05, - "C": 8.88e-05, - "V": 991.09056638, - "T": "2017-11-26T08:50:00", - "BV": 0.0877869 - }, - { - "O": 8.88e-05, - "H": 8.942e-05, - "L": 8.88e-05, - "C": 8.893e-05, - "V": 658.77935965, - "T": "2017-11-26T08:55:00", - "BV": 0.05874751 - }, - { - "O": 8.891e-05, - "H": 8.893e-05, - "L": 8.875e-05, - "C": 8.877e-05, - "V": 7920.73570705, - "T": "2017-11-26T09:00:00", - "BV": 0.7039405 - } - ] - - -@pytest.fixture -def ticker_history_without_bv(): - return [ - { - "O": 8.794e-05, - "H": 8.948e-05, - "L": 8.794e-05, - "C": 8.88e-05, - "V": 991.09056638, - "T": "2017-11-26T08:50:00" - }, - { - "O": 8.88e-05, - "H": 8.942e-05, - "L": 8.88e-05, - "C": 8.893e-05, - "V": 658.77935965, - "T": "2017-11-26T08:55:00" - }, - { - "O": 8.891e-05, - "H": 8.893e-05, - "L": 8.875e-05, - "C": 8.877e-05, - "V": 7920.73570705, - "T": "2017-11-26T09:00:00" - } - ] - - -@pytest.fixture -def result(): - with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) - - -@pytest.fixture -def default_strategy(): - strategy = Strategy() - strategy.init({'strategy': 'default_strategy'}) - return strategy - - -# FIX: -# Create an fixture/function -# that inserts a trade of some type and open-status -# return the open-order-id -# See tests in rpc/main that could use this diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py new file mode 100644 index 000000000..c9def6a9e --- /dev/null +++ b/freqtrade/tests/test_configuration.py @@ -0,0 +1,157 @@ +# pragma pylint: disable=protected-access, invalid-name, missing-docstring + +""" +Unit test file for configuration.py +""" +import json + +from copy import deepcopy +from unittest.mock import MagicMock +import pytest +from jsonschema import ValidationError + +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +import freqtrade.tests.conftest as tt # test tools + + +def test_configuration_object() -> None: + """ + Test the Constants object has the mandatory Constants + :return: None + """ + assert hasattr(Configuration, '_load_config') + assert hasattr(Configuration, '_load_config_file') + assert hasattr(Configuration, '_validate_config') + assert hasattr(Configuration, 'show_info') + assert hasattr(Configuration, 'get_config') + + +def test_load_config_invalid_pair(default_conf, mocker) -> None: + """ + Test the configuration validator with an invalid PAIR format + :param default_conf: Configuration already read from a file (JSON format) + :return: None + """ + mocker.patch.multiple( + 'freqtrade.configuration.Configuration', + _load_config=MagicMock(return_value=[]) + ) + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'].append('BTC-ETH') + + with pytest.raises(ValidationError, match=r'.*does not match.*'): + configuration = Configuration([]) + configuration._validate_config(conf) + + +def test_load_config_missing_attributes(default_conf, mocker) -> None: + """ + Test the configuration validator with a missing attribute + :param default_conf: Configuration already read from a file (JSON format) + :return: None + """ + mocker.patch.multiple( + 'freqtrade.configuration.Configuration', + _load_config=MagicMock(return_value=[]) + ) + conf = deepcopy(default_conf) + conf.pop('exchange') + + with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): + configuration = Configuration([]) + configuration._validate_config(conf) + + +def test_load_config_file(default_conf, mocker, caplog) -> None: + """ + Test _load_config_file() method + :return: + """ + mocker.patch.multiple( + 'freqtrade.configuration.Configuration', + _load_config=MagicMock(return_value=[]) + ) + file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + configuration = Configuration([]) + validated_conf = configuration._load_config_file('somefile') + assert file_mock.call_count == 1 + assert validated_conf.items() >= default_conf.items() + assert 'internals' in validated_conf + assert tt.log_has('Validating configuration ...', caplog.record_tuples) + + +def test_load_config(default_conf, mocker) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration._load_config() + + assert 'strategy' in validated_conf + assert validated_conf['strategy'] == 'default_strategy' + assert 'dynamic_whitelist' not in validated_conf + assert 'dry_run_db' not in validated_conf + + +def test_load_config_with_params(default_conf, mocker) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--dynamic-whitelist', '10', + '--strategy', 'test_strategy', + '--dry-run-db' + ] + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration._load_config() + + assert 'dynamic_whitelist' in validated_conf + assert validated_conf['dynamic_whitelist'] == 10 + assert 'strategy' in validated_conf + assert validated_conf['strategy'] == 'test_strategy' + assert 'dry_run_db' in validated_conf + assert validated_conf['dry_run_db'] is True + + +def test_show_info(default_conf, mocker, caplog) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--dynamic-whitelist', '10', + '--strategy', 'test_strategy', + '--dry-run-db' + ] + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + configuration.show_info() + + assert tt.log_has( + 'Using dynamically generated whitelist. (--dynamic-whitelist detected)', + caplog.record_tuples + ) + + assert tt.log_has( + 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". ' + '(--dry_run_db detected)', + caplog.record_tuples + ) + + # Test the Dry run condition + configuration.config.update({'dry_run': False}) + configuration.show_info() + assert tt.log_has( + 'Dry run is disabled. (--dry_run_db ignored)', + caplog.record_tuples + ) From e025dc0dba72519ced1992bda75ad04655b77820 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Feb 2018 23:33:54 -0800 Subject: [PATCH 05/56] Keep in misc file only tool functions --- freqtrade/misc.py | 418 +++-------------------------------- freqtrade/tests/test_misc.py | 209 +++--------------- 2 files changed, 59 insertions(+), 568 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 22041094c..741a306a1 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -1,30 +1,26 @@ -import argparse -import enum +""" +Various tool function for Freqtrade and scripts +""" + +import re import json import logging -import time -import os -import re from datetime import datetime -from typing import Any, Callable, Dict, List - import numpy as np -from jsonschema import Draft4Validator, validate -from jsonschema.exceptions import ValidationError, best_match -from wrapt import synchronized - -from freqtrade import __version__ logger = logging.getLogger(__name__) -class State(enum.Enum): - RUNNING = 0 - STOPPED = 1 - - -# Current application state -_STATE = State.STOPPED +def shorten_date(_date): + """ + Trim the date so it fits on small screens + """ + new_date = re.sub('seconds?', 'sec', _date) + new_date = re.sub('minutes?', 'min', new_date) + new_date = re.sub('hours?', 'h', new_date) + new_date = re.sub('days?', 'd', new_date) + new_date = re.sub('^an?', '1', new_date) + return new_date ############################################ @@ -32,7 +28,6 @@ _STATE = State.STOPPED # Matplotlib doesn't support ::datetime64, # # so we need to convert it into ::datetime # ############################################ - def datesarray_to_datetimearray(dates): """ Convert an pandas-array of timestamps into @@ -41,13 +36,18 @@ def datesarray_to_datetimearray(dates): """ times = [] dates = dates.astype(datetime) - for i in range(0, dates.size): - date = dates[i].to_pydatetime() + for index in range(0, dates.size): + date = dates[index].to_pydatetime() times.append(date) return np.array(times) def common_datearray(dfs): + """ + Return dates from Dataframe + :param dfs: Dataframe + :return: List of dates + """ alldates = {} for pair, pair_data in dfs.items(): dates = datesarray_to_datetimearray(pair_data['date']) @@ -61,375 +61,11 @@ def common_datearray(dfs): def file_dump_json(filename, data) -> None: - with open(filename, 'w') as fp: - json.dump(data, fp) - - -@synchronized -def update_state(state: State) -> None: """ - Updates the application state - :param state: new state - :return: None - """ - global _STATE - _STATE = state - - -@synchronized -def get_state() -> State: - """ - Gets the current application state + Dump JSON data into a file + :param filename: file to create + :param data: JSON Data to save :return: """ - return _STATE - - -def load_config(path: str) -> Dict: - """ - Loads a config file from the given path - :param path: path as str - :return: configuration as dictionary - """ - with open(path) as file: - conf = json.load(file) - if 'internals' not in conf: - conf['internals'] = {} - logger.info('Validating configuration ...') - try: - validate(conf, CONF_SCHEMA) - return conf - except ValidationError as exception: - logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception) - raise ValidationError( - best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message - ) - - -def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: - """ - Throttles the given callable that it - takes at least `min_secs` to finish execution. - :param func: Any callable - :param min_secs: minimum execution time in seconds - :return: Any - """ - start = time.time() - result = func(*args, **kwargs) - end = time.time() - duration = max(min_secs - (end - start), 0.0) - logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) - time.sleep(duration) - return result - - -def common_args_parser(description: str): - """ - Parses given common arguments and returns them as a parsed object. - """ - parser = argparse.ArgumentParser( - description=description - ) - parser.add_argument( - '-v', '--verbose', - help='be verbose', - action='store_const', - dest='loglevel', - const=logging.DEBUG, - default=logging.INFO, - ) - parser.add_argument( - '--version', - action='version', - version='%(prog)s {}'.format(__version__), - ) - parser.add_argument( - '-c', '--config', - help='specify configuration file (default: config.json)', - dest='config', - default='config.json', - type=str, - metavar='PATH', - ) - parser.add_argument( - '--datadir', - help='path to backtest data (default freqdata/tests/testdata)', - dest='datadir', - default=os.path.join('freqtrade', 'tests', 'testdata'), - type=str, - metavar='PATH', - ) - parser.add_argument( - '-s', '--strategy', - help='specify strategy file (default: freqtrade/strategy/default_strategy.py)', - dest='strategy', - default='default_strategy', - type=str, - metavar='PATH', - ) - return parser - - -def parse_args(args: List[str], description: str): - """ - Parses given arguments and returns an argparse Namespace instance. - Returns None if a sub command has been selected and executed. - """ - parser = common_args_parser(description) - parser.add_argument( - '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ - instead of memory DB. Work only if dry_run is enabled.', - action='store_true', - dest='dry_run_db', - ) - parser.add_argument( - '--dynamic-whitelist', - help='dynamically generate and update whitelist \ - based on 24h BaseVolume (Default 20 currencies)', # noqa - dest='dynamic_whitelist', - const=20, - type=int, - metavar='INT', - nargs='?', - ) - - build_subcommands(parser) - return parser.parse_args(args) - - -def scripts_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-p', '--pair', - help='Show profits for only this pairs. Pairs are comma-separated.', - dest='pair', - default=None - ) - - -def backtesting_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-l', '--live', - action='store_true', - dest='live', - help='using live data', - ) - parser.add_argument( - '-i', '--ticker-interval', - help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', - dest='ticker_interval', - type=int, - metavar='INT', - ) - parser.add_argument( - '--realistic-simulation', - help='uses max_open_trades from config to simulate real world limitations', - action='store_true', - dest='realistic_simulation', - ) - parser.add_argument( - '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \ - Use it if you want to run your backtesting with up-to-date data.', - action='store_true', - dest='refresh_pairs', - ) - parser.add_argument( - '--export', - help='Export backtest results, argument are: trades\ - Example --export=trades', - type=str, - default=None, - dest='export', - ) - parser.add_argument( - '--timerange', - help='Specify what timerange of data to use.', - default=None, - type=str, - dest='timerange', - ) - - -def hyperopt_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-e', '--epochs', - help='specify number of epochs (default: 100)', - dest='epochs', - default=100, - type=int, - metavar='INT', - ) - parser.add_argument( - '--use-mongodb', - help='parallelize evaluations with mongodb (requires mongod in PATH)', - dest='mongodb', - action='store_true', - ) - parser.add_argument( - '-i', '--ticker-interval', - help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', - dest='ticker_interval', - type=int, - metavar='INT', - ) - parser.add_argument( - '--timerange', - help='Specify what timerange of data to use.', - default=None, - type=str, - dest='timerange', - ) - - -def parse_timerange(text): - if text is None: - return None - syntax = [(r'^-(\d{8})$', (None, 'date')), - (r'^(\d{8})-$', ('date', None)), - (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^(-\d+)$', (None, 'line')), - (r'^(\d+)-$', ('line', None)), - (r'^(\d+)-(\d+)$', ('index', 'index'))] - for rex, stype in syntax: - # Apply the regular expression to text - match = re.match(rex, text) - if match: # Regex has matched - rvals = match.groups() - index = 0 - start = None - stop = None - if stype[0]: - start = rvals[index] - if stype[0] != 'date': - start = int(start) - index += 1 - if stype[1]: - stop = rvals[index] - if stype[1] != 'date': - stop = int(stop) - return (stype, start, stop) - raise Exception('Incorrect syntax for timerange "%s"' % text) - - -def build_subcommands(parser: argparse.ArgumentParser) -> None: - """ Builds and attaches all subcommands """ - from freqtrade.optimize import backtesting, hyperopt - - subparsers = parser.add_subparsers(dest='subparser') - - # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') - backtesting_cmd.set_defaults(func=backtesting.start) - backtesting_options(backtesting_cmd) - - # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') - hyperopt_cmd.set_defaults(func=hyperopt.start) - hyperopt_options(hyperopt_cmd) - - -# Required json-schema for user specified config -CONF_SCHEMA = { - 'type': 'object', - 'properties': { - 'max_open_trades': {'type': 'integer', 'minimum': 1}, - 'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, - 'stake_amount': {'type': 'number', 'minimum': 0.0005}, - 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', - 'CLP', 'CNY', 'CZK', 'DKK', - 'EUR', 'GBP', 'HKD', 'HUF', - 'IDR', 'ILS', 'INR', 'JPY', - 'KRW', 'MXN', 'MYR', 'NOK', - 'NZD', 'PHP', 'PKR', 'PLN', - 'RUB', 'SEK', 'SGD', 'THB', - 'TRY', 'TWD', 'ZAR', 'USD']}, - 'dry_run': {'type': 'boolean'}, - 'minimal_roi': { - 'type': 'object', - 'patternProperties': { - '^[0-9.]+$': {'type': 'number'} - }, - 'minProperties': 1 - }, - 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, - 'bid_strategy': { - 'type': 'object', - 'properties': { - 'ask_last_balance': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1, - 'exclusiveMaximum': False - }, - }, - 'required': ['ask_last_balance'] - }, - 'exchange': {'$ref': '#/definitions/exchange'}, - 'experimental': { - 'type': 'object', - 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'} - } - }, - 'telegram': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'token': {'type': 'string'}, - 'chat_id': {'type': 'string'}, - }, - 'required': ['enabled', 'token', 'chat_id'] - }, - 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, - 'internals': { - 'type': 'object', - 'properties': { - 'process_throttle_secs': {'type': 'number'}, - 'interval': {'type': 'integer'} - } - } - }, - 'definitions': { - 'exchange': { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'key': {'type': 'string'}, - 'secret': {'type': 'string'}, - 'pair_whitelist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - }, - 'pair_blacklist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - } - }, - 'required': ['name', 'key', 'secret', 'pair_whitelist'] - } - }, - 'anyOf': [ - {'required': ['exchange']} - ], - 'required': [ - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'fiat_display_currency', - 'dry_run', - 'bid_strategy', - 'telegram' - ] -} + with open(filename, 'w') as file: + json.dump(data, file) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index c0d4db7a1..95b9bf338 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,188 +1,31 @@ # pragma pylint: disable=missing-docstring,C0103 -import argparse -import json -import time -from copy import deepcopy -from unittest.mock import MagicMock + +""" +Unit test file for misc.py +""" import datetime -import pytest -from jsonschema import ValidationError -from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.misc import (common_args_parser, file_dump_json, load_config, - parse_args, parse_timerange, throttle, datesarray_to_datetimearray) +from unittest.mock import MagicMock +from freqtrade.analyze import Analyze +from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, file_dump_json) -def test_throttle(): - - def func(): - return 42 - - start = time.time() - result = throttle(func, min_secs=0.1) - end = time.time() - - assert result == 42 - assert end - start > 0.1 - - result = throttle(func, min_secs=-1) - assert result == 42 - - -def test_throttle_with_assets(): - - def func(nb_assets=-1): - return nb_assets - - result = throttle(func, min_secs=0.1, nb_assets=666) - assert result == 666 - - result = throttle(func, min_secs=0.1) - assert result == -1 - - -# Parse common command-line-arguments. Used for all tools - -def test_parse_args_none(): - args = common_args_parser('') - assert isinstance(args, argparse.ArgumentParser) - - -def test_parse_args_defaults(): - args = parse_args([], '') - assert args.config == 'config.json' - assert args.dynamic_whitelist is None - assert args.loglevel == 20 - - -def test_parse_args_config(): - args = parse_args(['-c', '/dev/null'], '') - assert args.config == '/dev/null' - - args = parse_args(['--config', '/dev/null'], '') - assert args.config == '/dev/null' - - -def test_parse_args_verbose(): - args = parse_args(['-v'], '') - assert args.loglevel == 10 - - args = parse_args(['--verbose'], '') - assert args.loglevel == 10 - - -def test_parse_args_version(): - with pytest.raises(SystemExit, match=r'0'): - parse_args(['--version'], '') - - -def test_parse_args_invalid(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['-c'], '') - - -# Parse command-line-arguments -# used for main, backtesting and hyperopt - - -def test_parse_args_dynamic_whitelist(): - args = parse_args(['--dynamic-whitelist'], '') - assert args.dynamic_whitelist == 20 - - -def test_parse_args_dynamic_whitelist_10(): - args = parse_args(['--dynamic-whitelist', '10'], '') - assert args.dynamic_whitelist == 10 - - -def test_parse_args_dynamic_whitelist_invalid_values(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['--dynamic-whitelist', 'abc'], '') - - -def test_parse_args_backtesting_invalid(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['backtesting --ticker-interval'], '') - - with pytest.raises(SystemExit, match=r'2'): - parse_args(['backtesting --ticker-interval', 'abc'], '') - - -def test_parse_args_backtesting_custom(): - args = [ - '-c', 'test_conf.json', - 'backtesting', - '--live', - '--ticker-interval', '1', - '--refresh-pairs-cached'] - call_args = parse_args(args, '') - assert call_args.config == 'test_conf.json' - assert call_args.live is True - assert call_args.loglevel == 20 - assert call_args.subparser == 'backtesting' - assert call_args.func is not None - assert call_args.ticker_interval == 1 - assert call_args.refresh_pairs is True - - -def test_parse_args_hyperopt_custom(): - args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20'] - call_args = parse_args(args, '') - assert call_args.config == 'test_conf.json' - assert call_args.epochs == 20 - assert call_args.loglevel == 20 - assert call_args.subparser == 'hyperopt' - assert call_args.func is not None - - -def test_file_dump_json(mocker): - file_open = mocker.patch('freqtrade.misc.open', MagicMock()) - json_dump = mocker.patch('json.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3]) - assert file_open.call_count == 1 - assert json_dump.call_count == 1 - - -def test_parse_timerange_incorrect(): - assert ((None, 'line'), None, -200) == parse_timerange('-200') - assert (('line', None), 200, None) == parse_timerange('200-') - with pytest.raises(Exception, match=r'Incorrect syntax.*'): - parse_timerange('-') - - -def test_load_config(default_conf, mocker): - file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open( - read_data=json.dumps(default_conf) - )) - validated_conf = load_config('somefile') - assert file_mock.call_count == 1 - assert validated_conf.items() >= default_conf.items() - - -def test_load_config_invalid_pair(default_conf, mocker): - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'].append('BTC-ETH') - mocker.patch( - 'freqtrade.misc.open', - mocker.mock_open( - read_data=json.dumps(conf))) - with pytest.raises(ValidationError, match=r'.*does not match.*'): - load_config('somefile') - - -def test_load_config_missing_attributes(default_conf, mocker): - conf = deepcopy(default_conf) - conf.pop('exchange') - mocker.patch( - 'freqtrade.misc.open', - mocker.mock_open( - read_data=json.dumps(conf))) - with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): - load_config('somefile') +def test_shorten_date() -> None: + """ + Test shorten_date() function + :return: None + """ + str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' + str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' + assert shorten_date(str_data) == str_shorten_data def test_datesarray_to_datetimearray(ticker_history): - dataframes = parse_ticker_dataframe(ticker_history) + """ + Test datesarray_to_datetimearray() function + :return: None + """ + dataframes = Analyze.parse_ticker_dataframe(ticker_history) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) @@ -194,3 +37,15 @@ def test_datesarray_to_datetimearray(ticker_history): date_len = len(dates) assert date_len == 3 + + +def test_file_dump_json(mocker): + """ + Test file_dump_json() + :return: None + """ + file_open = mocker.patch('freqtrade.misc.open', MagicMock()) + json_dump = mocker.patch('json.dump', MagicMock()) + file_dump_json('somefile', [1, 2, 3]) + assert file_open.call_count == 1 + assert json_dump.call_count == 1 From a8b8ab20b7c24009265f93b14641e87d611eeed5 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Feb 2018 00:28:02 -0800 Subject: [PATCH 06/56] Move Analyze to a class --- freqtrade/analyze.py | 247 ++++++++++++++++++++------------ freqtrade/tests/test_analyze.py | 145 ++++++++++++------- 2 files changed, 254 insertions(+), 138 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 70bf40936..a1ed09824 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -1,121 +1,190 @@ """ Functions to analyze ticker data with indicators and produce buy and sell signals """ -import logging -from datetime import timedelta -from enum import Enum -from typing import Dict, List - import arrow +from datetime import datetime, timedelta +from enum import Enum from pandas import DataFrame, to_datetime - +from typing import Dict, List from freqtrade.exchange import get_ticker_history +from freqtrade.logger import Logger from freqtrade.strategy.strategy import Strategy - -logger = logging.getLogger(__name__) +from freqtrade.persistence import Trade class SignalType(Enum): - """ Enum to distinguish between buy and sell signals """ + """ + Enum to distinguish between buy and sell signals + """ BUY = "buy" SELL = "sell" -def parse_ticker_dataframe(ticker: list) -> DataFrame: +class Analyze(object): """ - Analyses the trend for the given ticker history - :param ticker: See exchange.get_ticker_history - :return: DataFrame + Analyze class contains everything the bot need to determine if the situation is good for + buying or selling. """ - columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'} - frame = DataFrame(ticker) \ - .rename(columns=columns) - if 'BV' in frame: - frame.drop('BV', 1, inplace=True) - frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) - frame.sort_values('date', inplace=True) - return frame + def __init__(self, config: dict) -> None: + """ + Init Analyze + :param config: Bot configuration (use the one from Configuration()) + """ + self.logger = Logger(name=__name__).get_logger() + self.config = config + self.strategy = Strategy() + self.strategy.init(self.config) -def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame + @staticmethod + def parse_ticker_dataframe(ticker: list) -> DataFrame: + """ + Analyses the trend for the given ticker history + :param ticker: See exchange.get_ticker_history + :return: DataFrame + """ + columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'} + frame = DataFrame(ticker) \ + .rename(columns=columns) + if 'BV' in frame: + frame.drop('BV', 1, inplace=True) + frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) + frame.sort_values('date', inplace=True) + return frame - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - strategy = Strategy() - return strategy.populate_indicators(dataframe=dataframe) + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ -def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - strategy = Strategy() - return strategy.populate_buy_trend(dataframe=dataframe) + return self.strategy.populate_indicators(dataframe=dataframe) + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the buy signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + return self.strategy.populate_buy_trend(dataframe=dataframe) -def populate_sell_trend(dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - strategy = Strategy() - return strategy.populate_sell_trend(dataframe=dataframe) + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the sell signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + return self.strategy.populate_sell_trend(dataframe=dataframe) + def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: + """ + Parses the given ticker history and returns a populated DataFrame + add several TA indicators and buy signal to it + :return DataFrame with ticker data and indicator data + """ + dataframe = self.parse_ticker_dataframe(ticker_history) + dataframe = self.populate_indicators(dataframe) + dataframe = self.populate_buy_trend(dataframe) + dataframe = self.populate_sell_trend(dataframe) + return dataframe -def analyze_ticker(ticker_history: List[Dict]) -> DataFrame: - """ - Parses the given ticker history and returns a populated DataFrame - add several TA indicators and buy signal to it - :return DataFrame with ticker data and indicator data - """ - dataframe = parse_ticker_dataframe(ticker_history) - dataframe = populate_indicators(dataframe) - dataframe = populate_buy_trend(dataframe) - dataframe = populate_sell_trend(dataframe) - return dataframe + # FIX: Maybe return False, if an error has occured, + # Otherwise we might mask an error as an non-signal-scenario + def get_signal(self, pair: str, interval: int) -> (bool, bool): + """ + Calculates current signal based several technical analysis indicators + :param pair: pair in format BTC_ANT or BTC-ANT + :return: (Buy, Sell) A bool-tuple indicating buy/sell signal + """ + ticker_hist = get_ticker_history(pair, interval) + if not ticker_hist: + self.logger.warning('Empty ticker history for pair %s', pair) + return (False, False) # return False ? + try: + dataframe = self.analyze_ticker(ticker_hist) + except ValueError as error: + self.logger.warning( + 'Unable to analyze ticker for pair %s: %s', + pair, + str(error) + ) + return (False, False) # return False ? + except Exception as error: + self.logger.exception( + 'Unexpected error when analyzing ticker for pair %s: %s', + pair, + str(error) + ) + return (False, False) # return False ? -# FIX: Maybe return False, if an error has occured, -# Otherwise we might mask an error as an non-signal-scenario -def get_signal(pair: str, interval: int) -> (bool, bool): - """ - Calculates current signal based several technical analysis indicators - :param pair: pair in format BTC_ANT or BTC-ANT - :return: (Buy, Sell) A bool-tuple indicating buy/sell signal - """ - ticker_hist = get_ticker_history(pair, interval) - if not ticker_hist: - logger.warning('Empty ticker history for pair %s', pair) - return (False, False) # return False ? + if dataframe.empty: + self.logger.warning('Empty dataframe for pair %s', pair) + return (False, False) # return False ? - try: - dataframe = analyze_ticker(ticker_hist) - except ValueError as ex: - logger.warning('Unable to analyze ticker for pair %s: %s', pair, str(ex)) - return (False, False) # return False ? - except Exception as ex: - logger.exception('Unexpected error when analyzing ticker for pair %s: %s', pair, str(ex)) - return (False, False) # return False ? + latest = dataframe.iloc[-1] - if dataframe.empty: - logger.warning('Empty dataframe for pair %s', pair) - return (False, False) # return False ? + # Check if dataframe is out of date + signal_date = arrow.get(latest['date']) + if signal_date < arrow.now() - timedelta(minutes=(interval + 5)): + self.logger.warning('Too old dataframe for pair %s', pair) + return (False, False) # return False ? - latest = dataframe.iloc[-1] + (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 + self.logger.debug( + 'trigger: %s (pair=%s) buy=%s sell=%s', + latest['date'], + pair, + str(buy), + str(sell) + ) + return (buy, sell) - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - if signal_date < arrow.now() - timedelta(minutes=(interval + 5)): - logger.warning('Too old dataframe for pair %s', pair) - return (False, False) # return False ? + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: + """ + This function evaluate if on the condition required to trigger a sell has been reached + if the threshold is reached and updates the trade record. + :return: True if trade should be sold, False otherwise + """ + # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) + if self.min_roi_reached(trade=trade, current_rate=rate, current_time=date): + self.logger.debug('Executing sell due to ROI ...') + return True - (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 - logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell)) - return (buy, sell) + # Experimental: Check if the trade is profitable before selling it (avoid selling at loss) + if self.config.get('experimental', {}).get('sell_profit_only', False): + self.logger.debug('Checking if trade is profitable ...') + if trade.calc_profit(rate=rate) <= 0: + return False + + if sell and not buy and self.config.get('experimental', {}).get('use_sell_signal', False): + self.logger.debug('Executing sell due to sell signal ...') + return True + + return False + + def min_roi_reached(self, trade: Trade, current_rate: float, current_time: datetime) -> bool: + """ + Based an earlier trade and current price and ROI configuration, decides whether bot should + sell + :return True if bot should sell at current rate + """ + current_profit = trade.calc_profit_percent(current_rate) + if self.strategy.stoploss is not None and current_profit < float(self.strategy.stoploss): + self.logger.debug('Stop loss hit.') + return True + + # Check if time matches and current rate is above threshold + time_diff = (current_time - trade.open_date).total_seconds() / 60 + for duration, threshold in sorted(self.strategy.minimal_roi.items()): + if time_diff > float(duration) and current_profit > threshold: + return True + + self.logger.debug( + 'Threshold not reached. (cur_profit: %1.2f%%)', + float(current_profit) * 100.0 + ) + return False diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 41a6c1c2f..3c1687ab3 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -1,16 +1,45 @@ # pragma pylint: disable=missing-docstring, C0103 + +""" +Unit test file for analyse.py +""" + import datetime from unittest.mock import MagicMock - -import arrow import logging +import arrow from pandas import DataFrame import freqtrade.tests.conftest as tt # test tools -from freqtrade.analyze import (get_signal, parse_ticker_dataframe, - populate_buy_trend, populate_indicators, - populate_sell_trend) -from freqtrade.strategy.strategy import Strategy +from freqtrade.analyze import Analyze, SignalType + + +# Avoid to reinit the same object again and again +_ANALYZE = Analyze({'strategy': 'default_strategy'}) + + +def test_signaltype_object() -> None: + """ + Test the SignalType object has the mandatory Constants + :return: None + """ + assert hasattr(SignalType, 'BUY') + assert hasattr(SignalType, 'SELL') + + +def test_analyze_object() -> None: + """ + Test the Analyze object has the mandatory methods + :return: None + """ + assert hasattr(Analyze, 'parse_ticker_dataframe') + assert hasattr(Analyze, 'populate_indicators') + assert hasattr(Analyze, 'populate_buy_trend') + assert hasattr(Analyze, 'populate_sell_trend') + assert hasattr(Analyze, 'analyze_ticker') + assert hasattr(Analyze, 'get_signal') + assert hasattr(Analyze, 'should_sell') + assert hasattr(Analyze, 'min_roi_reached') def test_dataframe_correct_columns(result): @@ -18,71 +47,75 @@ def test_dataframe_correct_columns(result): ['close', 'high', 'low', 'open', 'date', 'volume'] -def test_dataframe_correct_length(result): - # no idea what this check truly does - should we just remove it? - assert len(result.index) == 14397 - - def test_populates_buy_trend(result): # Load the default strategy for the unit test, because this logic is done in main.py - Strategy().init({'strategy': 'default_strategy'}) - - dataframe = populate_buy_trend(populate_indicators(result)) + dataframe = _ANALYZE.populate_buy_trend(_ANALYZE.populate_indicators(result)) assert 'buy' in dataframe.columns def test_populates_sell_trend(result): # Load the default strategy for the unit test, because this logic is done in main.py - Strategy().init({'strategy': 'default_strategy'}) - - dataframe = populate_sell_trend(populate_indicators(result)) + dataframe = _ANALYZE.populate_sell_trend(_ANALYZE.populate_indicators(result)) assert 'sell' in dataframe.columns def test_returns_latest_buy_signal(mocker): mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) - ) - assert get_signal('BTC-ETH', 5) == (True, False) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) + ) ) - assert get_signal('BTC-ETH', 5) == (False, True) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (True, False) + + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) + ) + ) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (False, True) def test_returns_latest_sell_signal(mocker): mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) + ) ) - assert get_signal('BTC-ETH', 5) == (False, True) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (False, True) + + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) + ) ) - assert get_signal('BTC-ETH', 5) == (True, False) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (True, False) def test_get_signal_empty(default_conf, mocker, caplog): caplog.set_level(logging.INFO) mocker.patch('freqtrade.analyze.get_ticker_history', return_value=None) - assert (False, False) == get_signal('foo', int(default_conf['ticker_interval'])) - assert tt.log_has('Empty ticker history for pair foo', - caplog.record_tuples) + assert (False, False) == _ANALYZE.get_signal('foo', int(default_conf['ticker_interval'])) + assert tt.log_has('Empty ticker history for pair foo', caplog.record_tuples) def test_get_signal_exception_valueerror(default_conf, mocker, caplog): caplog.set_level(logging.INFO) mocker.patch('freqtrade.analyze.get_ticker_history', return_value=1) - mocker.patch('freqtrade.analyze.analyze_ticker', - side_effect=ValueError('xyz')) - assert (False, False) == get_signal('foo', int(default_conf['ticker_interval'])) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + side_effect=ValueError('xyz') + ) + ) + assert (False, False) == _ANALYZE.get_signal('foo', int(default_conf['ticker_interval'])) assert tt.log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) @@ -90,8 +123,13 @@ def test_get_signal_exception_valueerror(default_conf, mocker, caplog): def test_get_signal_empty_dataframe(default_conf, mocker, caplog): caplog.set_level(logging.INFO) mocker.patch('freqtrade.analyze.get_ticker_history', return_value=1) - mocker.patch('freqtrade.analyze.analyze_ticker', return_value=DataFrame([])) - assert (False, False) == get_signal('xyz', int(default_conf['ticker_interval'])) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([]) + ) + ) + assert (False, False) == _ANALYZE.get_signal('xyz', int(default_conf['ticker_interval'])) assert tt.log_has('Empty dataframe for pair xyz', caplog.record_tuples) @@ -102,27 +140,36 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): # FIX: The get_signal function has hardcoded 10, which we must inturn hardcode oldtime = arrow.utcnow() - datetime.timedelta(minutes=11) ticks = DataFrame([{'buy': 1, 'date': oldtime}]) - mocker.patch('freqtrade.analyze.analyze_ticker', return_value=DataFrame(ticks)) - assert (False, False) == get_signal('xyz', int(default_conf['ticker_interval'])) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame(ticks) + ) + ) + assert (False, False) == _ANALYZE.get_signal('xyz', int(default_conf['ticker_interval'])) assert tt.log_has('Too old dataframe for pair xyz', caplog.record_tuples) def test_get_signal_handles_exceptions(mocker): mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) - mocker.patch('freqtrade.analyze.analyze_ticker', - side_effect=Exception('invalid ticker history ')) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + side_effect=Exception('invalid ticker history ') + ) + ) - assert get_signal('BTC-ETH', 5) == (False, False) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (False, False) def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv): columns = ['close', 'high', 'low', 'open', 'date', 'volume'] # Test file with BV data - dataframe = parse_ticker_dataframe(ticker_history) + dataframe = Analyze.parse_ticker_dataframe(ticker_history) assert dataframe.columns.tolist() == columns # Test file without BV data - dataframe = parse_ticker_dataframe(ticker_history_without_bv) + dataframe = Analyze.parse_ticker_dataframe(ticker_history_without_bv) assert dataframe.columns.tolist() == columns From 4da033c7a2f058d75fdf5571eb6e9b57f41aa830 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Feb 2018 01:21:16 -0800 Subject: [PATCH 07/56] Refactor main.py - Update, clean, and improve code coverage on main.py - Move bot trading logic into Freqtradebot() class - Move unit tests to test_freqtradebot, add more coverage tests --- freqtrade/configuration.py | 2 + freqtrade/freqtradebot.py | 548 ++++++++++++ freqtrade/main.py | 577 +----------- freqtrade/tests/conftest.py | 131 +++ freqtrade/tests/test_freqtradebot.py | 1213 ++++++++++++++++++++++++++ freqtrade/tests/test_main.py | 874 ++----------------- freqtrade/tests/test_misc.py | 2 +- 7 files changed, 1997 insertions(+), 1350 deletions(-) create mode 100644 freqtrade/freqtradebot.py create mode 100644 freqtrade/tests/test_freqtradebot.py diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 72925e8ce..39368886b 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -21,12 +21,14 @@ class Configuration(object): self.args = args self.logger = Logger(name=__name__).get_logger() self.config = self._load_config() + self.show_info() def _load_config(self) -> Dict[str, Any]: """ Extract information for sys.argv and load the bot configuration :return: Configuration dictionary """ + self.logger.info('Using config: %s ...', self.args.config) config = self._load_config_file(self.args.config) # Add the strategy file to use diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py new file mode 100644 index 000000000..35acb7649 --- /dev/null +++ b/freqtrade/freqtradebot.py @@ -0,0 +1,548 @@ +""" +Freqtrade is the main module of this bot. It contains the class Freqtrade() +""" + +import arrow +import copy +import json +import requests +import time +import traceback +from cachetools import cached, TTLCache +from datetime import datetime +from typing import Dict, List, Optional, Any, Callable +from freqtrade.analyze import Analyze +from freqtrade.constants import Constants +from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.logger import Logger +from freqtrade.persistence import Trade +from freqtrade.rpc.rpc_manager import RPCManager +from freqtrade.state import State +from freqtrade import (DependencyException, OperationalException, exchange, persistence) + + +class FreqtradeBot(object): + """ + Freqtrade is the main class of the bot. + This is from here the bot start its logic. + """ + + def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None) -> bool: + """ + Init all variables and object the bot need to work + :param config: configuration dict, you can use the Configuration.get_config() + method to get the config dict. + :param db_url: database connector string for sqlalchemy (Optional) + """ + + # Init the logger + self.logger = Logger(name='freqtrade').get_logger() + + # Init bot states + self._state = State.STOPPED + + # Init objects + self.config = config + self.analyze = None + self.fiat_converter = None + self.rpc = None + self.persistence = None + self.exchange = None + + self._init_modules(db_url=db_url) + + def _init_modules(self, db_url: Optional[str] = None) -> None: + """ + Initializes all modules and updates the config + :param db_url: database connector string for sqlalchemy (Optional) + :return: None + """ + # Initialize all modules + self.analyze = Analyze(self.config) + self.fiat_converter = CryptoToFiatConverter() + self.rpc = RPCManager(self) + + persistence.init(self.config, db_url) + exchange.init(self.config) + + # Set initial application state + initial_state = self.config.get('initial_state') + + if initial_state: + self.update_state(State[initial_state.upper()]) + else: + self.update_state(State.STOPPED) + + def clean(self) -> bool: + """ + Cleanup the application state und finish all pending tasks + :return: None + """ + self.rpc.send_msg('*Status:* `Stopping trader...`') + self.logger.info('Stopping trader and cleaning up modules...') + self.update_state(State.STOPPED) + self.rpc.cleanup() + persistence.cleanup() + return True + + def update_state(self, state: State) -> None: + """ + Updates the application state + :param state: new state + :return: None + """ + self._state = state + + def get_state(self) -> State: + """ + Gets the current application state + :return: + """ + return self._state + + def worker(self, old_state: None) -> State: + """ + Trading routine that must be run at each loop + :param old_state: the previous service state from the previous call + :return: current service state + """ + new_state = self.get_state() + # Log state transition + if new_state != old_state: + self.rpc.send_msg('*Status:* `{}`'.format(new_state.name.lower())) + self.logger.info('Changing state to: %s', new_state.name) + + if new_state == State.STOPPED: + time.sleep(1) + elif new_state == State.RUNNING: + min_secs = self.config['internals'].get( + 'process_throttle_secs', + Constants.PROCESS_THROTTLE_SECS + ) + + nb_assets = self.config.get( + 'dynamic_whitelist', + Constants.DYNAMIC_WHITELIST + ) + + interval = int( + self.config.get( + 'ticker_interval', + Constants.TICKER_INTERVAL + ) + ) + + self._throttle(func=self._process, + min_secs=min_secs, + nb_assets=nb_assets, + interval=interval) + return new_state + + def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: + """ + Throttles the given callable that it + takes at least `min_secs` to finish execution. + :param func: Any callable + :param min_secs: minimum execution time in seconds + :return: Any + """ + start = time.time() + result = func(*args, **kwargs) + end = time.time() + duration = max(min_secs - (end - start), 0.0) + self.logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) + time.sleep(duration) + return result + + def _process(self, interval: int, nb_assets: Optional[int] = 0) -> bool: + """ + Queries the persistence layer for open trades and handles them, + otherwise a new trade is created. + :param: nb_assets: the maximum number of pairs to be traded at the same time + :return: True if one or more trades has been created or closed, False otherwise + """ + state_changed = False + try: + # Refresh whitelist based on wallet maintenance + sanitized_list = self._refresh_whitelist( + self._gen_pair_whitelist( + self.config['stake_currency'] + ) if nb_assets else self.config['exchange']['pair_whitelist'] + ) + + # Keep only the subsets of pairs wanted (up to nb_assets) + final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list + self.config['exchange']['pair_whitelist'] = final_list + + # Query trades from persistence layer + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + + # First process current opened trades + for trade in trades: + state_changed |= self.process_maybe_execute_sell(trade, interval) + + # Then looking for buy opportunities + if len(trades) < self.config['max_open_trades']: + state_changed = self.process_maybe_execute_buy(interval) + + if 'unfilledtimeout' in self.config: + # Check and handle any timed out open orders + self.check_handle_timedout(self.config['unfilledtimeout']) + Trade.session.flush() + + except (requests.exceptions.RequestException, json.JSONDecodeError) as error: + self.logger.warning( + 'Got %s in _process(), retrying in 30 seconds...', + error + ) + time.sleep(Constants.RETRY_TIMEOUT) + except OperationalException: + self.rpc.send_msg( + '*Status:* Got OperationalException:\n```\n{traceback}```{hint}' + .format( + traceback=traceback.format_exc(), + hint='Issue `/start` if you think it is safe to restart.' + ) + ) + self.logger.exception('Got OperationalException. Stopping trader ...') + self.update_state(State.STOPPED) + return state_changed + + @cached(TTLCache(maxsize=1, ttl=1800)) + def _gen_pair_whitelist(self, base_currency: str, key: str = 'BaseVolume') -> List[str]: + """ + Updates the whitelist with with a dynamically generated list + :param base_currency: base currency as str + :param key: sort key (defaults to 'BaseVolume') + :return: List of pairs + """ + summaries = sorted( + (s for s in exchange.get_market_summaries() if + s['MarketName'].startswith(base_currency)), + key=lambda s: s.get(key) or 0.0, + reverse=True + ) + + return [s['MarketName'].replace('-', '_') for s in summaries] + + def _refresh_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check wallet health and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + health = exchange.get_wallet_health() + known_pairs = set() + for status in health: + pair = '{}_{}'.format(self.config['stake_currency'], status['Currency']) + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.config['exchange'].get('pair_blacklist', []): + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not status['IsActive']: + sanitized_whitelist.remove(pair) + self.logger.info( + 'Ignoring %s from whitelist (reason: %s).', + pair, status.get('Notice') or 'wallet is not active' + ) + + # We need to remove pairs that are unknown + final_list = [x for x in sanitized_whitelist if x in known_pairs] + return final_list + + def get_target_bid(self, ticker: Dict[str, float]) -> float: + """ + Calculates bid target between current ask price and last price + :param ticker: Ticker to use for getting Ask and Last Price + :return: float: Price + """ + if ticker['ask'] < ticker['last']: + return ticker['ask'] + balance = self.config['bid_strategy']['ask_last_balance'] + return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + + # TODO: Remove the two parameters and use the value already in conf['stake_amount'] and + # int(conf['ticker_interval']) + def create_trade(self, stake_amount: float, interval: int) -> bool: + """ + Checks the implemented trading indicator(s) for a randomly picked pair, + if one pair triggers the buy_signal a new trade record gets created + :param stake_amount: amount of btc to spend + :param interval: Ticker interval used for Analyze + :return: True if a trade object has been created and persisted, False otherwise + """ + self.logger.info( + 'Checking buy signals to create a new trade with stake_amount: %f ...', + stake_amount + ) + whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) + # Check if stake_amount is fulfilled + if exchange.get_balance(self.config['stake_currency']) < stake_amount: + raise DependencyException( + 'stake amount is not fulfilled (currency={})'.format(self.config['stake_currency']) + ) + + # Remove currently opened and latest pairs from whitelist + for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): + if trade.pair in whitelist: + whitelist.remove(trade.pair) + self.logger.debug('Ignoring %s in pair whitelist', trade.pair) + + if not whitelist: + raise DependencyException('No pair in whitelist') + + # Pick pair based on StochRSI buy signals + for _pair in whitelist: + (buy, sell) = self.analyze.get_signal(_pair, interval) + if buy and not sell: + pair = _pair + break + else: + return False + + # Calculate amount + buy_limit = self.get_target_bid(exchange.get_ticker(pair)) + amount = stake_amount / buy_limit + + order_id = exchange.buy(pair, buy_limit, amount) + + stake_amount_fiat = self.fiat_converter.convert_amount( + stake_amount, + self.config['stake_currency'], + self.config['fiat_display_currency'] + ) + + # Create trade entity and return + self.rpc.send_msg( + '*{}:* Buying [{}]({}) with limit `{:.8f} ({:.6f} {}, {:.3f} {})` ' + .format( + exchange.get_name().upper(), + pair.replace('_', '/'), + exchange.get_pair_detail_url(pair), + buy_limit, + stake_amount, + self.config['stake_currency'], + stake_amount_fiat, + self.config['fiat_display_currency'] + ) + ) + # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL + trade = Trade( + pair=pair, + stake_amount=stake_amount, + amount=amount, + fee=exchange.get_fee(), + open_rate=buy_limit, + open_date=datetime.utcnow(), + exchange=exchange.get_name().upper(), + open_order_id=order_id + ) + Trade.session.add(trade) + Trade.session.flush() + return True + + def process_maybe_execute_buy(self, interval: int) -> bool: + """ + Tries to execute a buy trade in a safe way + :return: True if executed + """ + try: + # Create entity and execute trade + if self.create_trade(float(self.config['stake_amount']), interval): + return True + + self.logger.info( + 'Checked all whitelisted currencies. ' + 'Found no suitable entry positions for buying. Will keep looking ...' + ) + return False + except DependencyException as exception: + self.logger.warning('Unable to create trade: %s', exception) + return False + + def process_maybe_execute_sell(self, trade: Trade, interval: int) -> bool: + """ + Tries to execute a sell trade + :return: True if executed + """ + # Get order details for actual price per unit + if trade.open_order_id: + # Update trade with order values + self.logger.info('Got open order for %s', trade) + trade.update(exchange.get_order(trade.open_order_id)) + + if trade.is_open and trade.open_order_id is None: + # Check if we can sell our current pair + return self.handle_trade(trade, interval) + return False + + def handle_trade(self, trade: Trade, interval: int) -> bool: + """ + Sells the current pair if the threshold is reached and updates the trade record. + :return: True if trade has been sold, False otherwise + """ + if not trade.is_open: + raise ValueError('attempt to handle closed trade: {}'.format(trade)) + + self.logger.debug('Handling %s ...', trade) + current_rate = exchange.get_ticker(trade.pair)['bid'] + + (buy, sell) = (False, False) + + if self.config.get('experimental', {}).get('use_sell_signal'): + (buy, sell) = self.analyze.get_signal(trade.pair, interval) + + if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): + self.execute_sell(trade, current_rate) + return True + + return False + + def check_handle_timedout(self, timeoutvalue: int) -> None: + """ + Check if any orders are timed out and cancel if neccessary + :param timeoutvalue: Number of minutes until order is considered timed out + :return: None + """ + timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime + + for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): + try: + order = exchange.get_order(trade.open_order_id) + except requests.exceptions.RequestException: + self.logger.info( + 'Cannot query order for %s due to %s', + trade, + traceback.format_exc()) + continue + ordertime = arrow.get(order['opened']) + + # Check if trade is still actually open + if int(order['remaining']) == 0: + continue + + if order['type'] == "LIMIT_BUY" and ordertime < timeoutthreashold: + self.handle_timedout_limit_buy(trade, order) + elif order['type'] == "LIMIT_SELL" and ordertime < timeoutthreashold: + self.handle_timedout_limit_sell(trade, order) + + # FIX: 20180110, why is cancel.order unconditionally here, whereas + # it is conditionally called in the + # handle_timedout_limit_sell()? + def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool: + """Buy timeout - cancel order + :return: True if order was fully cancelled + """ + exchange.cancel_order(trade.open_order_id) + if order['remaining'] == order['amount']: + # if trade is not partially completed, just delete the trade + Trade.session.delete(trade) + # FIX? do we really need to flush, caller of + # check_handle_timedout will flush afterwards + Trade.session.flush() + self.logger.info('Buy order timeout for %s.', trade) + self.rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + return True + + # if trade is partially complete, edit the stake details for the trade + # and close the order + trade.amount = order['amount'] - order['remaining'] + trade.stake_amount = trade.amount * trade.open_rate + trade.open_order_id = None + self.logger.info('Partial buy order timeout for %s.', trade) + self.rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + return False + + # FIX: 20180110, should cancel_order() be cond. or unconditionally called? + def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> bool: + """ + Sell timeout - cancel order and update trade + :return: True if order was fully cancelled + """ + if order['remaining'] == order['amount']: + # if trade is not partially completed, just cancel the trade + exchange.cancel_order(trade.open_order_id) + trade.close_rate = None + trade.close_profit = None + trade.close_date = None + trade.is_open = True + trade.open_order_id = None + self.rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + self.logger.info('Sell order timeout for %s.', trade) + return True + + # TODO: figure out how to handle partially complete sell orders + return False + + def execute_sell(self, trade: Trade, limit: float) -> None: + """ + Executes a limit sell for the given trade and limit + :param trade: Trade instance + :param limit: limit rate for the sell order + :return: None + """ + # Execute sell and update trade record + order_id = exchange.sell(str(trade.pair), limit, trade.amount) + trade.open_order_id = order_id + + fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) + profit_trade = trade.calc_profit(rate=limit) + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + profit = trade.calc_profit_percent(current_rate) + + message = "*{exchange}:* Selling\n" \ + "*Current Pair:* [{pair}]({pair_url})\n" \ + "*Limit:* `{limit}`\n" \ + "*Amount:* `{amount}`\n" \ + "*Open Rate:* `{open_rate:.8f}`\n" \ + "*Current Rate:* `{current_rate:.8f}`\n" \ + "*Profit:* `{profit:.2f}%`" \ + "".format( + exchange=trade.exchange, + pair=trade.pair, + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit, + open_rate=trade.open_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + profit=round(profit * 100, 2), + ) + + # For regular case, when the configuration exists + if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: + fiat_converter = CryptoToFiatConverter() + profit_fiat = fiat_converter.convert_amount( + profit_trade, + self.config['stake_currency'], + self.config['fiat_display_currency'] + ) + message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ + '` / {profit_fiat:.3f} {fiat})`' \ + ''.format( + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade, + coin=self.config['stake_currency'], + profit_fiat=profit_fiat, + fiat=self.config['fiat_display_currency'], + ) + # Because telegram._forcesell does not have the configuration + # Ignore the FIAT value and does not show the stake_currency as well + else: + message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade + ) + + # Send the message + self.rpc.send_msg(message) + Trade.session.flush() diff --git a/freqtrade/main.py b/freqtrade/main.py index 163597eee..47e941c44 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -1,570 +1,73 @@ #!/usr/bin/env python3 -import copy -import json +""" +Main Freqtrade bot script. +Read the documentation to know what cli arguments you need. +""" + import logging import sys -import time -import traceback -from datetime import datetime -from typing import Dict, List, Optional, Any +from typing import Dict +from freqtrade.configuration import Configuration +from freqtrade.arguments import Arguments +from freqtrade.freqtradebot import FreqtradeBot +from freqtrade.logger import Logger +from freqtrade import (__version__) -import arrow -import requests -from cachetools import cached, TTLCache - -from freqtrade import (DependencyException, OperationalException, __version__, - exchange, persistence, rpc) -from freqtrade.analyze import get_signal -from freqtrade.fiat_convert import CryptoToFiatConverter -from freqtrade.misc import (State, get_state, load_config, parse_args, - throttle, update_state) -from freqtrade.persistence import Trade -from freqtrade.strategy.strategy import Strategy - -logger = logging.getLogger('freqtrade') - -_CONF: Dict[str, Any] = {} +logger = Logger(name='freqtrade').get_logger() -def refresh_whitelist(whitelist: List[str]) -> List[str]: +def main(sysargv: Dict) -> None: """ - Check wallet health and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to trade - :return: the list of pairs the user wants to trade without the one unavailable or black_listed - """ - sanitized_whitelist = whitelist - health = exchange.get_wallet_health() - known_pairs = set() - for status in health: - pair = '{}_{}'.format(_CONF['stake_currency'], status['Currency']) - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in _CONF['exchange'].get('pair_blacklist', []): - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not status['IsActive']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist (reason: %s).', - pair, status.get('Notice') or 'wallet is not active' - ) - - # We need to remove pairs that are unknown - final_list = [x for x in sanitized_whitelist if x in known_pairs] - return final_list - - -def process_maybe_execute_buy(interval: int) -> bool: - """ - Tries to execute a buy trade in a safe way - :return: True if executed - """ - try: - # Create entity and execute trade - if create_trade(float(_CONF['stake_amount']), interval): - return True - - logger.info( - 'Checked all whitelisted currencies. ' - 'Found no suitable entry positions for buying. Will keep looking ...' - ) - return False - except DependencyException as exception: - logger.warning('Unable to create trade: %s', exception) - return False - - -def process_maybe_execute_sell(trade: Trade, interval: int) -> bool: - """ - Tries to execute a sell trade - :return: True if executed - """ - # Get order details for actual price per unit - if trade.open_order_id: - # Update trade with order values - logger.info('Got open order for %s', trade) - trade.update(exchange.get_order(trade.open_order_id)) - - if trade.is_open and trade.open_order_id is None: - # Check if we can sell our current pair - return handle_trade(trade, interval) - return False - - -def _process(interval: int, nb_assets: Optional[int] = 0) -> bool: - """ - Queries the persistence layer for open trades and handles them, - otherwise a new trade is created. - :param: nb_assets: the maximum number of pairs to be traded at the same time - :return: True if one or more trades has been created or closed, False otherwise - """ - state_changed = False - try: - # Refresh whitelist based on wallet maintenance - sanitized_list = refresh_whitelist( - gen_pair_whitelist( - _CONF['stake_currency'] - ) if nb_assets else _CONF['exchange']['pair_whitelist'] - ) - - # Keep only the subsets of pairs wanted (up to nb_assets) - final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list - _CONF['exchange']['pair_whitelist'] = final_list - - # Query trades from persistence layer - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - - # First process current opened trades - for trade in trades: - state_changed |= process_maybe_execute_sell(trade, interval) - - # Then looking for buy opportunities - if len(trades) < _CONF['max_open_trades']: - state_changed = process_maybe_execute_buy(interval) - - if 'unfilledtimeout' in _CONF: - # Check and handle any timed out open orders - check_handle_timedout(_CONF['unfilledtimeout']) - Trade.session.flush() - - except (requests.exceptions.RequestException, json.JSONDecodeError) as error: - logger.warning( - 'Got %s in _process(), retrying in 30 seconds...', - error - ) - time.sleep(30) - except OperationalException: - rpc.send_msg('*Status:* Got OperationalException:\n```\n{traceback}```{hint}'.format( - traceback=traceback.format_exc(), - hint='Issue `/start` if you think it is safe to restart.' - )) - logger.exception('Got OperationalException. Stopping trader ...') - update_state(State.STOPPED) - return state_changed - - -# FIX: 20180110, why is cancel.order unconditionally here, whereas -# it is conditionally called in the -# handle_timedout_limit_sell()? -def handle_timedout_limit_buy(trade: Trade, order: Dict) -> bool: - """Buy timeout - cancel order - :return: True if order was fully cancelled - """ - exchange.cancel_order(trade.open_order_id) - if order['remaining'] == order['amount']: - # if trade is not partially completed, just delete the trade - Trade.session.delete(trade) - # FIX? do we really need to flush, caller of - # check_handle_timedout will flush afterwards - Trade.session.flush() - logger.info('Buy order timeout for %s.', trade) - rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - return True - - # if trade is partially complete, edit the stake details for the trade - # and close the order - trade.amount = order['amount'] - order['remaining'] - trade.stake_amount = trade.amount * trade.open_rate - trade.open_order_id = None - logger.info('Partial buy order timeout for %s.', trade) - rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - return False - - -# FIX: 20180110, should cancel_order() be cond. or unconditionally called? -def handle_timedout_limit_sell(trade: Trade, order: Dict) -> bool: - """ - Sell timeout - cancel order and update trade - :return: True if order was fully cancelled - """ - if order['remaining'] == order['amount']: - # if trade is not partially completed, just cancel the trade - exchange.cancel_order(trade.open_order_id) - trade.close_rate = None - trade.close_profit = None - trade.close_date = None - trade.is_open = True - trade.open_order_id = None - rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - logger.info('Sell order timeout for %s.', trade) - return True - - # TODO: figure out how to handle partially complete sell orders - return False - - -def check_handle_timedout(timeoutvalue: int) -> None: - """ - Check if any orders are timed out and cancel if neccessary - :param timeoutvalue: Number of minutes until order is considered timed out + This function will initiate the bot and start the trading loop. :return: None """ - timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime - - for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): - try: - order = exchange.get_order(trade.open_order_id) - except requests.exceptions.RequestException: - logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) - continue - ordertime = arrow.get(order['opened']) - - # Check if trade is still actually open - if int(order['remaining']) == 0: - continue - - if order['type'] == "LIMIT_BUY" and ordertime < timeoutthreashold: - handle_timedout_limit_buy(trade, order) - elif order['type'] == "LIMIT_SELL" and ordertime < timeoutthreashold: - handle_timedout_limit_sell(trade, order) - - -def execute_sell(trade: Trade, limit: float) -> None: - """ - Executes a limit sell for the given trade and limit - :param trade: Trade instance - :param limit: limit rate for the sell order - :return: None - """ - # Execute sell and update trade record - order_id = exchange.sell(str(trade.pair), limit, trade.amount) - trade.open_order_id = order_id - - fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) - profit_trade = trade.calc_profit(rate=limit) - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - profit = trade.calc_profit_percent(current_rate) - - message = """*{exchange}:* Selling -*Current Pair:* [{pair}]({pair_url}) -*Limit:* `{limit}` -*Amount:* `{amount}` -*Open Rate:* `{open_rate:.8f}` -*Current Rate:* `{current_rate:.8f}` -*Profit:* `{profit:.2f}%` - """.format( - exchange=trade.exchange, - pair=trade.pair, - pair_url=exchange.get_pair_detail_url(trade.pair), - limit=limit, - open_rate=trade.open_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - profit=round(profit * 100, 2), + arguments = Arguments( + sysargv, + 'Simple High Frequency Trading Bot for crypto currencies' ) + args = arguments.get_parsed_arg() - # For regular case, when the configuration exists - if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF: - fiat_converter = CryptoToFiatConverter() - profit_fiat = fiat_converter.convert_amount( - profit_trade, - _CONF['stake_currency'], - _CONF['fiat_display_currency'] - ) - message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ - '` / {profit_fiat:.3f} {fiat})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade, - coin=_CONF['stake_currency'], - profit_fiat=profit_fiat, - fiat=_CONF['fiat_display_currency'], - ) - # Because telegram._forcesell does not have the configuration - # Ignore the FIAT value and does not show the stake_currency as well - else: - message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade - ) - - # Send the message - rpc.send_msg(message) - Trade.session.flush() - - -def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) -> bool: - """ - Based an earlier trade and current price and ROI configuration, decides whether bot should sell - :return True if bot should sell at current rate - """ - strategy = Strategy() - - current_profit = trade.calc_profit_percent(current_rate) - if strategy.stoploss is not None and current_profit < float(strategy.stoploss): - logger.debug('Stop loss hit.') - return True - - # Check if time matches and current rate is above threshold - time_diff = (current_time - trade.open_date).total_seconds() / 60 - for duration, threshold in sorted(strategy.minimal_roi.items()): - if time_diff > float(duration) and current_profit > threshold: - return True - - logger.debug('Threshold not reached. (cur_profit: %1.2f%%)', float(current_profit) * 100.0) - return False - - -def should_sell(trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: - """ - This function evaluate if on the condition required to trigger a sell has been reached - if the threshold is reached and updates the trade record. - :return: True if trade should be sold, False otherwise - """ - # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) - if min_roi_reached(trade, rate, date): - logger.debug('Executing sell due to ROI ...') - return True - - # Experimental: Check if the trade is profitable before selling it (avoid selling at loss) - if _CONF.get('experimental', {}).get('sell_profit_only', False): - logger.debug('Checking if trade is profitable ...') - if trade.calc_profit(rate=rate) <= 0: - return False - - if sell and not buy and _CONF.get('experimental', {}).get('use_sell_signal', False): - logger.debug('Executing sell due to sell signal ...') - return True - - return False - - -def handle_trade(trade: Trade, interval: int) -> bool: - """ - Sells the current pair if the threshold is reached and updates the trade record. - :return: True if trade has been sold, False otherwise - """ - if not trade.is_open: - raise ValueError('attempt to handle closed trade: {}'.format(trade)) - - logger.debug('Handling %s ...', trade) - current_rate = exchange.get_ticker(trade.pair)['bid'] - - (buy, sell) = (False, False) - - if _CONF.get('experimental', {}).get('use_sell_signal'): - (buy, sell) = get_signal(trade.pair, interval) - - if should_sell(trade, current_rate, datetime.utcnow(), buy, sell): - execute_sell(trade, current_rate) - return True - - return False - - -def get_target_bid(ticker: Dict[str, float]) -> float: - """ Calculates bid target between current ask price and last price """ - if ticker['ask'] < ticker['last']: - return ticker['ask'] - balance = _CONF['bid_strategy']['ask_last_balance'] - return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) - - -def create_trade(stake_amount: float, interval: int) -> bool: - """ - Checks the implemented trading indicator(s) for a randomly picked pair, - if one pair triggers the buy_signal a new trade record gets created - :param stake_amount: amount of btc to spend - :return: True if a trade object has been created and persisted, False otherwise - """ - logger.info( - 'Checking buy signals to create a new trade with stake_amount: %f ...', - stake_amount - ) - whitelist = copy.deepcopy(_CONF['exchange']['pair_whitelist']) - # Check if stake_amount is fulfilled - if exchange.get_balance(_CONF['stake_currency']) < stake_amount: - raise DependencyException( - 'stake amount is not fulfilled (currency={})'.format(_CONF['stake_currency']) - ) - - # Remove currently opened and latest pairs from whitelist - for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): - if trade.pair in whitelist: - whitelist.remove(trade.pair) - logger.debug('Ignoring %s in pair whitelist', trade.pair) - if not whitelist: - raise DependencyException('No pair in whitelist') - - # Pick pair based on StochRSI buy signals - for _pair in whitelist: - (buy, sell) = get_signal(_pair, interval) - if buy and not sell: - pair = _pair - break - else: - return False - - # Calculate amount - buy_limit = get_target_bid(exchange.get_ticker(pair)) - amount = stake_amount / buy_limit - - order_id = exchange.buy(pair, buy_limit, amount) - - fiat_converter = CryptoToFiatConverter() - stake_amount_fiat = fiat_converter.convert_amount( - stake_amount, - _CONF['stake_currency'], - _CONF['fiat_display_currency'] - ) - - # Create trade entity and return - rpc.send_msg('*{}:* Buying [{}]({}) with limit `{:.8f} ({:.6f} {}, {:.3f} {})` '.format( - exchange.get_name().upper(), - pair.replace('_', '/'), - exchange.get_pair_detail_url(pair), - buy_limit, stake_amount, _CONF['stake_currency'], - stake_amount_fiat, _CONF['fiat_display_currency'] - )) - # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL - trade = Trade( - pair=pair, - stake_amount=stake_amount, - amount=amount, - fee=exchange.get_fee(), - open_rate=buy_limit, - open_date=datetime.utcnow(), - exchange=exchange.get_name().upper(), - open_order_id=order_id - ) - Trade.session.add(trade) - Trade.session.flush() - return True - - -def init(config: dict, db_url: Optional[str] = None) -> None: - """ - Initializes all modules and updates the config - :param config: config as dict - :param db_url: database connector string for sqlalchemy (Optional) - :return: None - """ - # Initialize all modules - rpc.init(config) - persistence.init(config, db_url) - exchange.init(config) - - strategy = Strategy() - strategy.init(config) - - # Set initial application state - initial_state = config.get('initial_state') - if initial_state: - update_state(State[initial_state.upper()]) - else: - update_state(State.STOPPED) - - -@cached(TTLCache(maxsize=1, ttl=1800)) -def gen_pair_whitelist(base_currency: str, key: str = 'BaseVolume') -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'BaseVolume') - :return: List of pairs - """ - summaries = sorted( - (s for s in exchange.get_market_summaries() if s['MarketName'].startswith(base_currency)), - key=lambda s: s.get(key) or 0.0, - reverse=True - ) - - return [s['MarketName'].replace('-', '_') for s in summaries] - - -def cleanup() -> None: - """ - Cleanup the application state und finish all pending tasks - :return: None - """ - rpc.send_msg('*Status:* `Stopping trader...`') - logger.info('Stopping trader and cleaning up modules...') - update_state(State.STOPPED) - persistence.cleanup() - rpc.cleanup() - exit(0) - - -def main(sysargv=sys.argv[1:]) -> int: - """ - Loads and validates the config and handles the main loop - :return: None - """ - global _CONF - args = parse_args(sysargv, - 'Simple High Frequency Trading Bot for crypto currencies') - - # A subcommand has been issued + # A subcommand has been issued. + # Means if Backtesting or Hyperopt have been called we exit the bot if hasattr(args, 'func'): args.func(args) return 0 - # Initialize logger - logging.basicConfig( - level=args.loglevel, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) - logger.info( 'Starting freqtrade %s (loglevel=%s)', __version__, logging.getLevelName(args.loglevel) ) - # Load and validate configuration - _CONF = load_config(args.config) - - # Add the strategy file to use - _CONF.update({'strategy': args.strategy}) - - # Initialize all modules and start main loop - if args.dynamic_whitelist: - logger.info('Using dynamically generated whitelist. (--dynamic-whitelist detected)') - - # If the user ask for Dry run with a local DB instead of memory - if args.dry_run_db: - if _CONF.get('dry_run', False): - _CONF.update({'dry_run_db': True}) - logger.info( - 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". (--dry_run_db detected)' - ) - else: - logger.info('Dry run is disabled. (--dry_run_db ignored)') - try: - init(_CONF) - old_state = None + # Load and validate configuration + configuration = Configuration(args) - while True: - new_state = get_state() - # Log state transition - if new_state != old_state: - rpc.send_msg('*Status:* `{}`'.format(new_state.name.lower())) - logger.info('Changing state to: %s', new_state.name) + # Init the bot + freqtrade = FreqtradeBot(configuration.get_config()) + + state = None + while 1: + state = freqtrade.worker(old_state=state) - if new_state == State.STOPPED: - time.sleep(1) - elif new_state == State.RUNNING: - throttle( - _process, - min_secs=_CONF['internals'].get('process_throttle_secs', 10), - nb_assets=args.dynamic_whitelist, - interval=int(_CONF.get('ticker_interval', 5)) - ) - old_state = new_state except KeyboardInterrupt: logger.info('Got SIGINT, aborting ...') except BaseException: logger.exception('Got fatal exception!') finally: - cleanup() - return 0 + freqtrade.clean() + sys.exit(0) + + +def set_loggers() -> None: + """ + Set the logger level for Third party libs + :return: None + """ + logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) + logging.getLogger('telegram').setLevel(logging.INFO) if __name__ == '__main__': + set_loggers() main(sys.argv[1:]) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index e2967b845..26a61e923 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -281,3 +281,134 @@ def default_strategy(): # that inserts a trade of some type and open-status # return the open-order-id # See tests in rpc/main that could use this + + +@pytest.fixture +def get_market_summaries_data(): + """ + This fixture is a real result from exchange.get_market_summaries() but reduced to only + 8 entries. 4 BTC, 4 USTD + :return: JSON market summaries + """ + return [ + { + 'Ask': 1.316e-05, + 'BaseVolume': 5.72599471, + 'Bid': 1.3e-05, + 'Created': '2014-04-14T00:00:00', + 'High': 1.414e-05, + 'Last': 1.298e-05, + 'Low': 1.282e-05, + 'MarketName': 'BTC-XWC', + 'OpenBuyOrders': 2000, + 'OpenSellOrders': 1484, + 'PrevDay': 1.376e-05, + 'TimeStamp': '2018-02-05T01:32:40.493', + 'Volume': 424041.21418375 + }, + { + 'Ask': 0.00627051, + 'BaseVolume': 93.23302388, + 'Bid': 0.00618192, + 'Created': '2016-10-20T04:48:30.387', + 'High': 0.00669897, + 'Last': 0.00618192, + 'Low': 0.006, + 'MarketName': 'BTC-XZC', + 'OpenBuyOrders': 343, + 'OpenSellOrders': 2037, + 'PrevDay': 0.00668229, + 'TimeStamp': '2018-02-05T01:32:43.383', + 'Volume': 14863.60730702 + }, + { + 'Ask': 0.01137247, + 'BaseVolume': 383.55922657, + 'Bid': 0.01136006, + 'Created': '2016-11-15T20:29:59.73', + 'High': 0.012, + 'Last': 0.01137247, + 'Low': 0.01119883, + 'MarketName': 'BTC-ZCL', + 'OpenBuyOrders': 1332, + 'OpenSellOrders': 5317, + 'PrevDay': 0.01179603, + 'TimeStamp': '2018-02-05T01:32:42.773', + 'Volume': 33308.07358285 + }, + { + 'Ask': 0.04155821, + 'BaseVolume': 274.75369074, + 'Bid': 0.04130002, + 'Created': '2016-10-28T17:13:10.833', + 'High': 0.04354429, + 'Last': 0.041585, + 'Low': 0.0413, + 'MarketName': 'BTC-ZEC', + 'OpenBuyOrders': 863, + 'OpenSellOrders': 5579, + 'PrevDay': 0.0429, + 'TimeStamp': '2018-02-05T01:32:43.21', + 'Volume': 6479.84033259 + }, + { + 'Ask': 210.99999999, + 'BaseVolume': 615132.70989532, + 'Bid': 210.05503736, + 'Created': '2017-07-21T01:08:49.397', + 'High': 257.396, + 'Last': 211.0, + 'Low': 209.05333589, + 'MarketName': 'USDT-XMR', + 'OpenBuyOrders': 180, + 'OpenSellOrders': 1203, + 'PrevDay': 247.93528899, + 'TimeStamp': '2018-02-05T01:32:43.117', + 'Volume': 2688.17410793 + }, + { + 'Ask': 0.79589979, + 'BaseVolume': 9349557.01853031, + 'Bid': 0.789226, + 'Created': '2017-07-14T17:10:10.737', + 'High': 0.977, + 'Last': 0.79589979, + 'Low': 0.781, + 'MarketName': 'USDT-XRP', + 'OpenBuyOrders': 1075, + 'OpenSellOrders': 6508, + 'PrevDay': 0.93300218, + 'TimeStamp': '2018-02-05T01:32:42.383', + 'Volume': 10801663.00788851 + }, + { + 'Ask': 0.05154982, + 'BaseVolume': 2311087.71232136, + 'Bid': 0.05040107, + 'Created': '2017-12-29T19:29:18.357', + 'High': 0.06668561, + 'Last': 0.0508, + 'Low': 0.05006731, + 'MarketName': 'USDT-XVG', + 'OpenBuyOrders': 655, + 'OpenSellOrders': 5544, + 'PrevDay': 0.0627, + 'TimeStamp': '2018-02-05T01:32:41.507', + 'Volume': 40031424.2152716 + }, + { + 'Ask': 332.65500022, + 'BaseVolume': 562911.87455665, + 'Bid': 330.00000001, + 'Created': '2017-07-14T17:10:10.673', + 'High': 401.59999999, + 'Last': 332.65500019, + 'Low': 330.0, + 'MarketName': 'USDT-ZEC', + 'OpenBuyOrders': 161, + 'OpenSellOrders': 1731, + 'PrevDay': 391.42, + 'TimeStamp': '2018-02-05T01:32:42.947', + 'Volume': 1571.09647946 + } + ] diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py new file mode 100644 index 000000000..5177bd3b7 --- /dev/null +++ b/freqtrade/tests/test_freqtradebot.py @@ -0,0 +1,1213 @@ +# pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments + +""" +Unit test file for freqtradebot.py +""" + +import logging +import time +from unittest.mock import MagicMock +from copy import deepcopy +from typing import Optional +import arrow +import pytest +import requests +from sqlalchemy import create_engine + +import freqtrade.tests.conftest as tt # test tools +from freqtrade import DependencyException, OperationalException +from freqtrade.exchange import Exchanges +from freqtrade.freqtradebot import FreqtradeBot +from freqtrade.state import State +from freqtrade.persistence import Trade + + +# Functions for recurrent object patching +def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: + """ + This function patch _init_modules() to not call dependencies + :param mocker: a Mocker object to apply patches + :param config: Config to pass to the bot + :return: None + """ + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + patch_pymarketcap(mocker) + + return FreqtradeBot(config) + + +def patch_get_signal(mocker, value=(True, False)) -> None: + """ + + :param mocker: mocker to patch Analyze class + :param value: which value Analyze.get_signal() must return + :return: None + """ + mocker.patch( + 'freqtrade.freqtradebot.Analyze.get_signal', + side_effect=lambda s, t: value + ) + + +def patch_RPCManager(mocker) -> MagicMock: + """ + This function mock RPC manager to avoid repeating this code in almost every tests + :param mocker: mocker to patch RPCManager class + :return: RPCManager.send_msg MagicMock to track if this method is called + """ + mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) + rpc_mock = mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) + return rpc_mock + + +def patch_pymarketcap(mocker, value: Optional[str] = None) -> None: + """ + Mocker to Pymarketcap to speed up tests + :param mocker: mocker to patch Pymarketcap class + :return: None + """ + pymarketcap = MagicMock() + + if value: + pymarketcap.ticker = {'price_usd': 12345.0} + + mocker.patch('freqtrade.fiat_convert.Pymarketcap', pymarketcap) + + +# Unit tests +def test_freqtradebot_object() -> None: + """ + Test the FreqtradeBot object has the mandatory public methods + """ + assert hasattr(FreqtradeBot, 'worker') + assert hasattr(FreqtradeBot, 'get_state') + assert hasattr(FreqtradeBot, 'update_state') + assert hasattr(FreqtradeBot, 'clean') + assert hasattr(FreqtradeBot, 'create_trade') + assert hasattr(FreqtradeBot, 'get_target_bid') + assert hasattr(FreqtradeBot, 'process_maybe_execute_buy') + assert hasattr(FreqtradeBot, 'process_maybe_execute_sell') + assert hasattr(FreqtradeBot, 'handle_trade') + assert hasattr(FreqtradeBot, 'check_handle_timedout') + assert hasattr(FreqtradeBot, 'handle_timedout_limit_buy') + assert hasattr(FreqtradeBot, 'handle_timedout_limit_sell') + assert hasattr(FreqtradeBot, 'execute_sell') + + +def test_freqtradebot(mocker, default_conf) -> None: + """ + Test __init__, _init_modules, update_state, and get_state methods + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.get_state() is State.RUNNING + + conf = deepcopy(default_conf) + conf.pop('initial_state') + freqtrade = FreqtradeBot(conf) + assert freqtrade.get_state() is State.STOPPED + + +@pytest.mark.skip(reason="Test not implemented") +def test_clean() -> None: + """ + Test clean() method + """ + pass + + +@pytest.mark.skip(reason="Test not implemented") +def test_worker() -> None: + """ + Test worker() method + """ + pass + + +def test_throttle(mocker, default_conf, caplog) -> None: + """ + Test _throttle() method + """ + def func(): + """ + Test function to throttle + """ + return 42 + + caplog.set_level(logging.DEBUG) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + start = time.time() + result = freqtrade._throttle(func, min_secs=0.1) + end = time.time() + + assert result == 42 + assert end - start > 0.1 + assert tt.log_has('Throttling func for 0.10 seconds', caplog.record_tuples) + + result = freqtrade._throttle(func, min_secs=-1) + assert result == 42 + + +def test_throttle_with_assets(mocker, default_conf) -> None: + """ + Test _throttle() method when the function passed can have parameters + """ + def func(nb_assets=-1): + """ + Test function to throttle + """ + return nb_assets + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + result = freqtrade._throttle(func, min_secs=0.1, nb_assets=666) + assert result == 666 + + result = freqtrade._throttle(func, min_secs=0.1) + assert result == -1 + + +@pytest.mark.skip(reason="Test not implemented") +def test_process() -> None: + """ + Test _process() method + """ + pass + + +def test_gen_pair_whitelist(mocker, default_conf, get_market_summaries_data) -> None: + """ + Test _gen_pair_whitelist() method + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_market_summaries', + return_value=get_market_summaries_data + ) + + # Test to retrieved BTC sorted on BaseVolume + whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC') + assert whitelist == ['BTC_ZCL', 'BTC_ZEC', 'BTC_XZC', 'BTC_XWC'] + + # Test to retrieved BTC sorted on OpenBuyOrders + whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='OpenBuyOrders') + assert whitelist == ['BTC_XWC', 'BTC_ZCL', 'BTC_ZEC', 'BTC_XZC'] + + # Test with USDT sorted on BaseVolume + whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') + assert whitelist == ['USDT_XRP', 'USDT_XVG', 'USDT_XMR', 'USDT_ZEC'] + + # Test with ETH (our fixture does not have ETH, but Bittrex returns them) + whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') + assert whitelist == [] + + +@pytest.mark.skip(reason="Test not implemented") +def test_refresh_whitelist() -> None: + """ + Test _refresh_whitelist() method + """ + pass + + +def test_create_trade(default_conf, ticker, limit_buy_order, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + # Save state of current whitelist + whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == Exchanges.BITTREX.name + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + assert trade.open_rate == 0.00001099 + assert trade.amount == 90.99181073 + + assert whitelist == default_conf['exchange']['pair_whitelist'] + + +def test_create_trade_minimal_amount(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + buy_mock = MagicMock(return_value='mocked_limit_buy') + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=buy_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + min_stake_amount = 0.0005 + freqtrade.create_trade(min_stake_amount, int(default_conf['ticker_interval'])) + rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] + assert rate * amount >= min_stake_amount + + +def test_create_trade_no_stake_amount(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy'), + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + with pytest.raises(DependencyException, match=r'.*stake amount.*'): + freqtrade.create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) + + +def test_create_trade_no_pairs(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ["BTC_ETH"] + conf['exchange']['pair_blacklist'] = ["BTC_ETH"] + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + with pytest.raises(DependencyException, match=r'.*No pair in whitelist.*'): + freqtrade.create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) + + +def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ["BTC_ETH"] + conf['exchange']['pair_blacklist'] = ["BTC_ETH"] + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + with pytest.raises(DependencyException, match=r'.*No pair in whitelist.*'): + freqtrade.create_trade(conf['stake_amount'], int(conf['ticker_interval'])) + + +def test_create_trade_no_signal(default_conf, mocker) -> None: + """ + Test create_trade() method + """ + conf = deepcopy(default_conf) + conf['dry_run'] = True + + patch_get_signal(mocker, value=(False, False)) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker_history=MagicMock(return_value=20), + get_balance=MagicMock(return_value=20) + ) + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + stake_amount = 10 + Trade.query = MagicMock() + Trade.query.filter = MagicMock() + assert not freqtrade.create_trade(stake_amount, int(conf['ticker_interval'])) + + +def test_process_trade_creation(default_conf, ticker, limit_buy_order, + health, mocker, caplog) -> None: + """ + Test the trade creation in _process() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(return_value='mocked_limit_buy'), + get_order=MagicMock(return_value=limit_buy_order) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert not trades + + result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + assert result is True + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert len(trades) == 1 + trade = trades[0] + assert trade is not None + assert trade.stake_amount == default_conf['stake_amount'] + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == Exchanges.BITTREX.name + assert trade.open_rate == 0.00001099 + assert trade.amount == 90.99181073703367 + + assert tt.log_has( + 'Checking buy signals to create a new trade with stake_amount: 0.001000 ...', + caplog.record_tuples + ) + + +def test_process_exchange_failures(default_conf, ticker, health, mocker) -> None: + """ + Test _process() method when a RequestException happens + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(side_effect=requests.exceptions.RequestException) + ) + sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + assert result is False + assert sleep_mock.has_calls() + + +def test_process_operational_exception(default_conf, ticker, health, mocker) -> None: + """ + Test _process() method when an OperationalException happens + """ + patch_get_signal(mocker) + msg_mock = patch_RPCManager(mocker) + patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(side_effect=OperationalException) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + assert freqtrade.get_state() == State.RUNNING + + result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + assert result is False + assert freqtrade.get_state() == State.STOPPED + assert 'OperationalException' in msg_mock.call_args_list[-1][0][0] + + +def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, mocker) -> None: + """ + Test _process() + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(return_value='mocked_limit_buy'), + get_order=MagicMock(return_value=limit_buy_order) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert not trades + result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + assert result is True + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert len(trades) == 1 + + result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + assert result is False + + +@pytest.mark.skip(reason="Test not implemented") +def test_get_target_bid(): + """ + Test get_target_bid() method + """ + pass + + +def test_balance_fully_ask_side(mocker) -> None: + """ + Test get_target_bid() method + """ + freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 0.0}}) + + assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20 + + +def test_balance_fully_last_side(mocker) -> None: + """ + Test get_target_bid() method + """ + freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 1.0}}) + + assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10 + + +def test_balance_bigger_last_ask(mocker) -> None: + """ + Test get_target_bid() method + """ + freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 1.0}}) + + assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5 + + +def test_process_maybe_execute_buy(mocker, default_conf) -> None: + """ + Test process_maybe_execute_buy() method + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=True)) + assert freqtrade.process_maybe_execute_buy(int(default_conf['ticker_interval'])) + + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=False)) + assert not freqtrade.process_maybe_execute_buy(int(default_conf['ticker_interval'])) + + +def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> None: + """ + Test exception on process_maybe_execute_buy() method + """ + caplog.set_level(logging.INFO) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mocker.patch( + 'freqtrade.freqtradebot.FreqtradeBot.create_trade', + MagicMock(side_effect=DependencyException) + ) + freqtrade.process_maybe_execute_buy(int(default_conf['ticker_interval'])) + tt.log_has('Unable to create trade:', caplog.record_tuples) + + +def test_process_maybe_execute_sell(mocker, default_conf) -> None: + """ + Test process_maybe_execute_sell() method + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) + mocker.patch('freqtrade.freqtradebot.exchange.get_order', return_value=1) + + trade = MagicMock() + trade.open_order_id = '123' + assert not freqtrade.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) + trade.is_open = True + trade.open_order_id = None + # Assert we call handle_trade() if trade is feasible for execution + assert freqtrade.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) + + +def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test check_handle() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value='mocked_limit_buy'), + sell=MagicMock(return_value='mocked_limit_sell') + ) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.Pymarketcap._cache_symbols', return_value={'BTC': 1}) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade + + trade.update(limit_buy_order) + assert trade.is_open is True + + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True + assert trade.open_order_id == 'mocked_limit_sell' + + # Simulate fulfilled LIMIT_SELL order for trade + trade.update(limit_sell_order) + + assert trade.close_rate == 0.00001173 + assert trade.close_profit == 0.06201057 + assert trade.calc_profit() == 0.00006217 + assert trade.close_date is not None + + +def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: + """ + Test check_handle() method + """ + conf = deepcopy(default_conf) + conf.update({'experimental': {'use_sell_signal': True}}) + + patch_get_signal(mocker, value=(True, True)) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + # Buy and Sell triggering, so doing nothing ... + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 0 + + # Buy is triggering, so buying ... + patch_get_signal(mocker, value=(True, False)) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 1 + assert trades[0].is_open is True + + # Buy and Sell are not triggering, so doing nothing ... + patch_get_signal(mocker, value=(False, False)) + assert freqtrade.handle_trade(trades[0], int(default_conf['ticker_interval'])) is False + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 1 + assert trades[0].is_open is True + + # Buy and Sell are triggering, so doing nothing ... + patch_get_signal(mocker, value=(True, True)) + assert freqtrade.handle_trade(trades[0], int(default_conf['ticker_interval'])) is False + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 1 + assert trades[0].is_open is True + + # Sell is triggering, guess what : we are Selling! + patch_get_signal(mocker, value=(False, True)) + trades = Trade.query.all() + assert freqtrade.handle_trade(trades[0], int(default_conf['ticker_interval'])) is True + + +def test_handle_trade_roi(default_conf, ticker, mocker, caplog) -> None: + """ + Test check_handle() method + """ + caplog.set_level(logging.DEBUG) + conf = deepcopy(default_conf) + conf.update({'experimental': {'use_sell_signal': True}}) + + patch_get_signal(mocker, value=(True, False)) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + trade.is_open = True + + # FIX: sniffing logs, suggest handle_trade should not execute_sell + # instead that responsibility should be moved out of handle_trade(), + # we might just want to check if we are in a sell condition without + # executing + # if ROI is reached we must sell + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade, interval=int(default_conf['ticker_interval'])) + assert tt.log_has('Executing sell due to ROI ...', caplog.record_tuples) + + +def test_handle_trade_experimental(default_conf, ticker, mocker, caplog) -> None: + """ + Test check_handle() method + """ + caplog.set_level(logging.DEBUG) + conf = deepcopy(default_conf) + conf.update({'experimental': {'use_sell_signal': True}}) + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + trade.is_open = True + + patch_get_signal(mocker, value=(False, False)) + assert not freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) + + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) + assert tt.log_has('Executing sell due to sell signal ...', caplog.record_tuples) + + +def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test check_handle() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create trade and sell it + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade + + trade.update(limit_buy_order) + trade.update(limit_sell_order) + assert trade.is_open is False + + with pytest.raises(ValueError, match=r'.*closed trade.*'): + freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) + + +def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker) -> None: + """ + Test check_handle_timedout() method + """ + rpc_mock = patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(return_value=limit_buy_order_old), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_buy = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + + # check it does cancel buy orders over the time limit + freqtrade.check_handle_timedout(600) + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 0 + + +def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: + """ + Test check_handle_timedout() method + """ + rpc_mock = patch_RPCManager(mocker) + patch_pymarketcap(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(return_value=limit_sell_order_old), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_sell = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(hours=-5).datetime, + close_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=False + ) + + Trade.session.add(trade_sell) + + # check it does cancel sell orders over the time limit + freqtrade.check_handle_timedout(600) + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + assert trade_sell.is_open is True + + +def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, + mocker) -> None: + """ + Test check_handle_timedout() method + """ + rpc_mock = patch_RPCManager(mocker) + patch_pymarketcap(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(return_value=limit_buy_order_old_partial), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_buy = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + + # check it does cancel buy orders over the time limit + # note this is for a partially-complete buy order + freqtrade.check_handle_timedout(600) + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() + assert len(trades) == 1 + assert trades[0].amount == 23.0 + assert trades[0].stake_amount == trade_buy.open_rate * trades[0].amount + + +def test_handle_timedout_limit_buy(mocker, default_conf) -> None: + """ + Test handle_timedout_limit_buy() method + """ + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + cancel_order=cancel_order_mock + ) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + Trade.session = MagicMock() + trade = MagicMock() + order = {'remaining': 1, + 'amount': 1} + assert freqtrade.handle_timedout_limit_buy(trade, order) + assert cancel_order_mock.call_count == 1 + order['amount'] = 2 + assert not freqtrade.handle_timedout_limit_buy(trade, order) + assert cancel_order_mock.call_count == 2 + + +def test_handle_timedout_limit_sell(mocker, default_conf) -> None: + """ + Test handle_timedout_limit_sell() method + """ + patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + cancel_order=cancel_order_mock + ) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade = MagicMock() + order = {'remaining': 1, + 'amount': 1} + assert freqtrade.handle_timedout_limit_sell(trade, order) + assert cancel_order_mock.call_count == 1 + order['amount'] = 2 + assert not freqtrade.handle_timedout_limit_sell(trade, order) + # Assert cancel_order was not called (callcount remains unchanged) + assert cancel_order_mock.call_count == 1 + + +def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker) -> None: + """ + Test execute_sell() method with a ticker going UP + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert 'Profit' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker) -> None: + """ + Test execute_sell() method with a ticker going DOWN + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down + ) + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_without_conf_sell_up(default_conf, ticker, ticker_sell_up, mocker) -> None: + """ + Test execute_sell() method with a ticker going DOWN and with a bot config empty + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) + freqtrade.config = {} + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert '(profit: 6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] + assert 'USD' not in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_without_conf_sell_down(default_conf, ticker, + ticker_sell_down, mocker) -> None: + """ + Test execute_sell() method with a ticker going DOWN and with a bot config empty + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down + ) + + freqtrade.config = {} + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + + +def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when enabled + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00002172, + 'ask': 0.00002173, + 'last': 0.00002172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': True, + } + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True + + +def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when disabled + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00002172, + 'ask': 0.00002173, + 'last': 0.00002172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': False, + } + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True + + +def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': True, + } + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is False + + +def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': False, + } + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 1adfa8418..ae89912e3 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,30 +1,22 @@ -# pragma pylint: disable=missing-docstring, C0103 -import copy +""" +Unit test file for main.py +""" + import logging from unittest.mock import MagicMock - -import arrow import pytest -import requests -from sqlalchemy import create_engine -import freqtrade.main as main +from freqtrade.main import main, set_loggers import freqtrade.tests.conftest as tt # test tools -from freqtrade import DependencyException, OperationalException -from freqtrade.exchange import Exchanges -from freqtrade.main import (_process, check_handle_timedout, create_trade, - execute_sell, get_target_bid, handle_trade, init) -from freqtrade.misc import State, get_state -from freqtrade.persistence import Trade -def test_parse_args_backtesting(mocker): - """ Test that main() can start backtesting or hyperopt. - and also ensure we can pass some specific arguments - further argument parsing is done in test_misc.py """ - backtesting_mock = mocker.patch( - 'freqtrade.optimize.backtesting.start', MagicMock()) - main.main(['backtesting']) +def test_parse_args_backtesting(mocker) -> None: + """ + Test that main() can start backtesting and also ensure we can pass some specific arguments + further argument parsing is done in test_arguments.py + """ + backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock()) + main(['backtesting']) assert backtesting_mock.call_count == 1 call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' @@ -35,10 +27,12 @@ def test_parse_args_backtesting(mocker): assert call_args.ticker_interval is None -def test_main_start_hyperopt(mocker): - hyperopt_mock = mocker.patch( - 'freqtrade.optimize.hyperopt.start', MagicMock()) - main.main(['hyperopt']) +def test_main_start_hyperopt(mocker) -> None: + """ + Test that main() can start hyperopt. + """ + hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock()) + main(['hyperopt']) assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' @@ -47,795 +41,51 @@ def test_main_start_hyperopt(mocker): assert call_args.func is not None -def test_process_maybe_execute_buy(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.create_trade', return_value=True) - assert main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) - mocker.patch('freqtrade.main.create_trade', return_value=False) - assert not main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) - - -def test_process_maybe_execute_sell(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.handle_trade', return_value=True) - mocker.patch('freqtrade.exchange.get_order', return_value=1) - trade = MagicMock() - trade.open_order_id = '123' - assert not main.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) - trade.is_open = True - trade.open_order_id = None - # Assert we call handle_trade() if trade is feasible for execution - assert main.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) - - -def test_process_maybe_execute_buy_exception(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.create_trade', MagicMock(side_effect=DependencyException)) - main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) - tt.log_has('Unable to create trade:', caplog.record_tuples) - - -def test_process_trade_creation(default_conf, ticker, limit_buy_order, health, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(return_value='mocked_limit_buy'), - get_order=MagicMock(return_value=limit_buy_order)) - init(default_conf, create_engine('sqlite://')) - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert not trades - - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is True - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert len(trades) == 1 - trade = trades[0] - assert trade is not None - assert trade.stake_amount == default_conf['stake_amount'] - assert trade.is_open - assert trade.open_date is not None - assert trade.exchange == Exchanges.BITTREX.name - assert trade.open_rate == 0.00001099 - assert trade.amount == 90.99181073703367 - - -def test_process_exchange_failures(default_conf, ticker, health, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(side_effect=requests.exceptions.RequestException)) - init(default_conf, create_engine('sqlite://')) - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is False - assert sleep_mock.has_calls() - - -def test_process_operational_exception(default_conf, ticker, health, mocker): - msg_mock = MagicMock() - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=msg_mock) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(side_effect=OperationalException)) - init(default_conf, create_engine('sqlite://')) - assert get_state() == State.RUNNING - - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is False - assert get_state() == State.STOPPED - assert 'OperationalException' in msg_mock.call_args_list[-1][0][0] - - -def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(return_value='mocked_limit_buy'), - get_order=MagicMock(return_value=limit_buy_order)) - init(default_conf, create_engine('sqlite://')) - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert not trades - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is True - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert len(trades) == 1 - - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is False - - -def test_create_trade(default_conf, ticker, limit_buy_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - # Save state of current whitelist - whitelist = copy.deepcopy(default_conf['exchange']['pair_whitelist']) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade is not None - assert trade.stake_amount == 0.001 - assert trade.is_open - assert trade.open_date is not None - assert trade.exchange == Exchanges.BITTREX.name - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - - assert trade.open_rate == 0.00001099 - assert trade.amount == 90.99181073 - - assert whitelist == default_conf['exchange']['pair_whitelist'] - - -def test_create_trade_minimal_amount(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - buy_mock = mocker.patch( - 'freqtrade.main.exchange.buy', MagicMock(return_value='mocked_limit_buy') - ) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - min_stake_amount = 0.0005 - create_trade(min_stake_amount, int(default_conf['ticker_interval'])) - rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] - assert rate * amount >= min_stake_amount - - -def test_create_trade_no_stake_amount(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy'), - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5)) - with pytest.raises(DependencyException, match=r'.*stake amount.*'): - create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) - - -def test_create_trade_no_pairs(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - - with pytest.raises(DependencyException, match=r'.*No pair in whitelist.*'): - conf = copy.deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = [] - mocker.patch.dict('freqtrade.main._CONF', conf) - create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) - - -def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - - with pytest.raises(DependencyException, match=r'.*No pair in whitelist.*'): - conf = copy.deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ["BTC_ETH"] - conf['exchange']['pair_blacklist'] = ["BTC_ETH"] - mocker.patch.dict('freqtrade.main._CONF', conf) - create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) - - -def test_create_trade_no_signal(default_conf, mocker): - default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', MagicMock(return_value=(False, False))) - mocker.patch.multiple('freqtrade.exchange', - get_ticker_history=MagicMock(return_value=20)) - mocker.patch.multiple('freqtrade.main.exchange', - get_balance=MagicMock(return_value=20)) - stake_amount = 10 - Trade.query = MagicMock() - Trade.query.filter = MagicMock() - assert not create_trade(stake_amount, int(default_conf['ticker_interval'])) - - -def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 - }), - buy=MagicMock(return_value='mocked_limit_buy'), - sell=MagicMock(return_value='mocked_limit_sell')) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - trade.update(limit_buy_order) - assert trade.is_open is True - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True - assert trade.open_order_id == 'mocked_limit_sell' - - # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) - - assert trade.close_rate == 0.00001173 - assert trade.close_profit == 0.06201057 - assert trade.calc_profit() == 0.00006217 - assert trade.close_date is not None - - -def test_handle_overlpapping_signals(default_conf, ticker, mocker): - default_conf.update({'experimental': {'use_sell_signal': True}}) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - # Buy and Sell triggering, so doing nothing ... - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 0 - - # Buy is triggering, so buying ... - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - create_trade(0.001, int(default_conf['ticker_interval'])) - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 1 - assert trades[0].is_open is True - - # Buy and Sell are not triggering, so doing nothing ... - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, False)) - assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is False - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 1 - assert trades[0].is_open is True - - # Buy and Sell are triggering, so doing nothing ... - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is False - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 1 - assert trades[0].is_open is True - - # Sell is triggering, guess what : we are Selling! - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - trades = Trade.query.all() - assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is True - - -def test_handle_trade_roi(default_conf, ticker, mocker, caplog): - caplog.set_level(logging.DEBUG) - default_conf.update({'experimental': {'use_sell_signal': True}}) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - mocker.patch('freqtrade.main.min_roi_reached', return_value=True) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.is_open = True - - # FIX: sniffing logs, suggest handle_trade should not execute_sell - # instead that responsibility should be moved out of handle_trade(), - # we might just want to check if we are in a sell condition without - # executing - # if ROI is reached we must sell - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, interval=int(default_conf['ticker_interval'])) - assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples - # if ROI is reached we must sell even if sell-signal is not signalled - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, interval=int(default_conf['ticker_interval'])) - assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples - - -def test_handle_trade_experimental(default_conf, ticker, mocker, caplog): - caplog.set_level(logging.DEBUG) - default_conf.update({'experimental': {'use_sell_signal': True}}) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.is_open = True - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, False)) - value_returned = handle_trade(trade, int(default_conf['ticker_interval'])) - assert value_returned is False - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) - s = 'Executing sell due to sell signal ...' - assert ('freqtrade', logging.DEBUG, s) in caplog.record_tuples - - -def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - - # Create trade and sell it - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - trade.update(limit_buy_order) - trade.update(limit_sell_order) - assert trade.is_open is False - - with pytest.raises(ValueError, match=r'.*closed trade.*'): - handle_trade(trade, int(default_conf['ticker_interval'])) - - -def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - cancel_order_mock = MagicMock() - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_order=MagicMock(return_value=limit_buy_order_old), - cancel_order=cancel_order_mock) - init(default_conf, create_engine('sqlite://')) - - trade_buy = Trade( - pair='BTC_ETH', - open_rate=0.00001099, - exchange='BITTREX', - open_order_id='123456789', - amount=90.99181073, - fee=0.0, - stake_amount=1, - open_date=arrow.utcnow().shift(minutes=-601).datetime, - is_open=True +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + previous_value1 = logging.getLogger('requests.packages.urllib3').level + previous_value2 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests.packages.urllib3').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('telegram').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + +def test_main(mocker, caplog) -> None: + """ + Test main() function. + In this test we are skipping the while True loop by throwing an exception. + """ + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + _init_modules=MagicMock(), + worker=MagicMock( + side_effect=KeyboardInterrupt + ), + clean=MagicMock(), ) - Trade.session.add(trade_buy) + # Test Main + the KeyboardInterrupt exception + with pytest.raises(SystemExit) as pytest_wrapped_e: + main([]) + tt.log_has('Starting freqtrade', caplog.record_tuples) + tt.log_has('Got SIGINT, aborting ...', caplog.record_tuples) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 42 - # check it does cancel buy orders over the time limit - check_handle_timedout(600) - assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 - trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() - nb_trades = len(trades) - assert nb_trades == 0 - - -def test_handle_timedout_limit_buy(mocker): - cancel_order = MagicMock() - mocker.patch('freqtrade.exchange.cancel_order', cancel_order) - Trade.session = MagicMock() - trade = MagicMock() - order = {'remaining': 1, - 'amount': 1} - assert main.handle_timedout_limit_buy(trade, order) - assert cancel_order.call_count == 1 - order['amount'] = 2 - assert not main.handle_timedout_limit_buy(trade, order) - assert cancel_order.call_count == 2 - - -def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - cancel_order_mock = MagicMock() - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_order=MagicMock(return_value=limit_sell_order_old), - cancel_order=cancel_order_mock) - init(default_conf, create_engine('sqlite://')) - - trade_sell = Trade( - pair='BTC_ETH', - open_rate=0.00001099, - exchange='BITTREX', - open_order_id='123456789', - amount=90.99181073, - fee=0.0, - stake_amount=1, - open_date=arrow.utcnow().shift(hours=-5).datetime, - close_date=arrow.utcnow().shift(minutes=-601).datetime, - is_open=False + # Test the BaseException case + mocker.patch( + 'freqtrade.freqtradebot.FreqtradeBot.worker', + MagicMock(side_effect=BaseException) ) - - Trade.session.add(trade_sell) - - # check it does cancel sell orders over the time limit - check_handle_timedout(600) - assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 - assert trade_sell.is_open is True - - -def test_handle_timedout_limit_sell(mocker): - cancel_order = MagicMock() - mocker.patch('freqtrade.exchange.cancel_order', cancel_order) - trade = MagicMock() - order = {'remaining': 1, - 'amount': 1} - assert main.handle_timedout_limit_sell(trade, order) - assert cancel_order.call_count == 1 - order['amount'] = 2 - assert not main.handle_timedout_limit_sell(trade, order) - # Assert cancel_order was not called (callcount remains unchanged) - assert cancel_order.call_count == 1 - - -def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, - mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - cancel_order_mock = MagicMock() - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_order=MagicMock(return_value=limit_buy_order_old_partial), - cancel_order=cancel_order_mock) - init(default_conf, create_engine('sqlite://')) - - trade_buy = Trade( - pair='BTC_ETH', - open_rate=0.00001099, - exchange='BITTREX', - open_order_id='123456789', - amount=90.99181073, - fee=0.0, - stake_amount=1, - open_date=arrow.utcnow().shift(minutes=-601).datetime, - is_open=True - ) - - Trade.session.add(trade_buy) - - # check it does cancel buy orders over the time limit - # note this is for a partially-complete buy order - check_handle_timedout(600) - assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 - trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() - assert len(trades) == 1 - assert trades[0].amount == 23.0 - assert trades[0].stake_amount == trade_buy.open_rate * trades[0].amount - - -def test_balance_fully_ask_side(mocker): - mocker.patch.dict('freqtrade.main._CONF', {'bid_strategy': {'ask_last_balance': 0.0}}) - assert get_target_bid({'ask': 20, 'last': 10}) == 20 - - -def test_balance_fully_last_side(mocker): - mocker.patch.dict('freqtrade.main._CONF', {'bid_strategy': {'ask_last_balance': 1.0}}) - assert get_target_bid({'ask': 20, 'last': 10}) == 10 - - -def test_balance_bigger_last_ask(mocker): - mocker.patch.dict('freqtrade.main._CONF', {'bid_strategy': {'ask_last_balance': 1.0}}) - assert get_target_bid({'ask': 5, 'last': 10}) == 5 - - -def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Increase the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - - execute_sell(trade=trade, limit=ticker_sell_up()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert 'Profit' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] - assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Decrease the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_down) - - execute_sell(trade=trade, limit=ticker_sell_down()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_execute_sell_without_conf_sell_down(default_conf, ticker, ticker_sell_down, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Decrease the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_down) - mocker.patch('freqtrade.main._CONF', {}) - - execute_sell(trade=trade, limit=ticker_sell_down()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - - -def test_execute_sell_without_conf_sell_up(default_conf, ticker, ticker_sell_up, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Increase the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - mocker.patch('freqtrade.main._CONF', {}) - - execute_sell(trade=trade, limit=ticker_sell_up()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert '(profit: 6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] - assert 'USD' not in rpc_mock.call_args_list[-1][0][0] - - -def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': True, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00002172, - 'ask': 0.00002173, - 'last': 0.00002172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True - - -def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': False, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00002172, - 'ask': 0.00002173, - 'last': 0.00002172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True - - -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': True, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is False - - -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': False, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True + with pytest.raises(SystemExit): + main([]) + tt.log_has('Got fatal exception!', caplog.record_tuples) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 95b9bf338..e51144496 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -39,7 +39,7 @@ def test_datesarray_to_datetimearray(ticker_history): assert date_len == 3 -def test_file_dump_json(mocker): +def test_file_dump_json(mocker) -> None: """ Test file_dump_json() :return: None From db67b106059803b72586b034e435c2cf7d2e844f Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 6 Feb 2018 20:22:17 -0800 Subject: [PATCH 08/56] Remove Singleton from Strategy() --- freqtrade/analyze.py | 13 +++++++++--- freqtrade/constants.py | 1 + freqtrade/strategy/strategy.py | 25 ++++++----------------- freqtrade/tests/strategy/test_strategy.py | 16 +++++---------- freqtrade/tests/test_constants.py | 15 +++++++------- 5 files changed, 30 insertions(+), 40 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index a1ed09824..f3126d723 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -33,8 +33,7 @@ class Analyze(object): self.logger = Logger(name=__name__).get_logger() self.config = config - self.strategy = Strategy() - self.strategy.init(self.config) + self.strategy = Strategy(self.config) @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: @@ -60,7 +59,6 @@ class Analyze(object): you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. """ - return self.strategy.populate_indicators(dataframe=dataframe) def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: @@ -97,6 +95,7 @@ class Analyze(object): """ Calculates current signal based several technical analysis indicators :param pair: pair in format BTC_ANT or BTC-ANT + :param interval: Interval to use (in min) :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ ticker_hist = get_ticker_history(pair, interval) @@ -188,3 +187,11 @@ class Analyze(object): float(current_profit) * 100.0 ) return False + + + def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: + """ + Creates a dataframe and populates indicators for given ticker data + """ + return {pair: self.populate_indicators(self.parse_ticker_dataframe(pair_data)) + for pair, pair_data in tickerdata.items()} \ No newline at end of file diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 765a4fe3e..a3f91d774 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -14,6 +14,7 @@ class Constants(object): TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec + DEFAULT_STRATEGY = 'default_strategy' # Required json-schema for user specified config CONF_SCHEMA = { diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 7ea8e81ac..8333beea7 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -3,12 +3,12 @@ """ This module load custom strategies """ +import importlib import os import sys -import logging -import importlib - from pandas import DataFrame +from freqtrade.logger import Logger +from freqtrade.constants import Constants from freqtrade.strategy.interface import IStrategy @@ -19,32 +19,19 @@ class Strategy(object): """ This class contains all the logic to load custom strategy class """ - __instance = None - - DEFAULT_STRATEGY = 'default_strategy' - - def __new__(cls) -> object: - """ - Used to create the Singleton - :return: Strategy object - """ - if Strategy.__instance is None: - Strategy.__instance = object.__new__(cls) - return Strategy.__instance - - def init(self, config: dict) -> None: + def __init__(self, config: dict={}) -> None: """ Load the custom class from config parameter :param config: :return: """ - self.logger = logging.getLogger(__name__) + self.logger = Logger(name=__name__).get_logger() # Verify the strategy is in the configuration, otherwise fallback to the default strategy if 'strategy' in config: strategy = config['strategy'] else: - strategy = self.DEFAULT_STRATEGY + strategy = Constants.DEFAULT_STRATEGY # Load the strategy self._load_strategy(strategy) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 890718d60..8c099b502 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -21,7 +21,6 @@ def test_search_strategy(): def test_strategy_structure(): - assert hasattr(Strategy, 'init') assert hasattr(Strategy, 'populate_indicators') assert hasattr(Strategy, 'populate_buy_trend') assert hasattr(Strategy, 'populate_sell_trend') @@ -53,8 +52,7 @@ def test_load_not_found_strategy(caplog): def test_strategy(result): - strategy = Strategy() - strategy.init({'strategy': 'default_strategy'}) + strategy = Strategy({'strategy': 'default_strategy'}) assert hasattr(strategy.custom_strategy, 'minimal_roi') assert strategy.minimal_roi['0'] == 0.04 @@ -82,8 +80,7 @@ def test_strategy_override_minimal_roi(caplog): "0": 0.5 } } - strategy = Strategy() - strategy.init(config) + strategy = Strategy(config) assert hasattr(strategy.custom_strategy, 'minimal_roi') assert strategy.minimal_roi['0'] == 0.5 @@ -99,8 +96,7 @@ def test_strategy_override_stoploss(caplog): 'strategy': 'default_strategy', 'stoploss': -0.5 } - strategy = Strategy() - strategy.init(config) + strategy = Strategy(config) assert hasattr(strategy.custom_strategy, 'stoploss') assert strategy.stoploss == -0.5 @@ -117,8 +113,7 @@ def test_strategy_override_ticker_interval(caplog): 'strategy': 'default_strategy', 'ticker_interval': 60 } - strategy = Strategy() - strategy.init(config) + strategy = Strategy(config) assert hasattr(strategy.custom_strategy, 'ticker_interval') assert strategy.ticker_interval == 60 @@ -138,8 +133,7 @@ def test_strategy_fallback_default_strategy(): def test_strategy_singleton(): - strategy1 = Strategy() - strategy1.init({'strategy': 'default_strategy'}) + strategy1 = Strategy({'strategy': 'default_strategy'}) assert hasattr(strategy1.custom_strategy, 'minimal_roi') assert strategy1.minimal_roi['0'] == 0.04 diff --git a/freqtrade/tests/test_constants.py b/freqtrade/tests/test_constants.py index e50fbb880..4c94926bc 100644 --- a/freqtrade/tests/test_constants.py +++ b/freqtrade/tests/test_constants.py @@ -10,13 +10,14 @@ def test_constant_object() -> None: Test the Constants object has the mandatory Constants :return: None """ - constant = Constants() - assert hasattr(constant, 'CONF_SCHEMA') - assert hasattr(constant, 'DYNAMIC_WHITELIST') - assert hasattr(constant, 'PROCESS_THROTTLE_SECS') - assert hasattr(constant, 'TICKER_INTERVAL') - assert hasattr(constant, 'HYPEROPT_EPOCH') - assert hasattr(constant, 'RETRY_TIMEOUT') + assert hasattr(Constants, 'CONF_SCHEMA') + assert hasattr(Constants, 'DYNAMIC_WHITELIST') + assert hasattr(Constants, 'PROCESS_THROTTLE_SECS') + assert hasattr(Constants, 'TICKER_INTERVAL') + assert hasattr(Constants, 'HYPEROPT_EPOCH') + assert hasattr(Constants, 'RETRY_TIMEOUT') + assert hasattr(Constants, 'DEFAULT_STRATEGY') + def test_conf_schema() -> None: From 1d251d61511886d34ff5c290594d2e4cb3044e94 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 8 Feb 2018 23:35:38 -0800 Subject: [PATCH 09/56] Move Backtesting to a class and add unit tests --- freqtrade/configuration.py | 48 ++ freqtrade/logger.py | 44 +- freqtrade/optimize/__init__.py | 35 +- freqtrade/optimize/backtesting.py | 504 +++++++++++-------- freqtrade/tests/conftest.py | 7 - freqtrade/tests/optimize/test_backtesting.py | 447 +++++++++++----- freqtrade/tests/optimize/test_optimize.py | 87 ++-- freqtrade/tests/test_configuration.py | 144 +++++- freqtrade/tests/test_logger.py | 53 +- 9 files changed, 942 insertions(+), 427 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 39368886b..38925bd4e 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -42,6 +42,9 @@ class Configuration(object): if self.args.dry_run_db and config.get('dry_run', False): config.update({'dry_run_db': True}) + # Load Backtesting / Hyperopt + config = self._load_backtesting_config(config) + return config def _load_config_file(self, path: str) -> Dict[str, Any]: @@ -59,6 +62,51 @@ class Configuration(object): return self._validate_config(conf) + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load Backtesting and Hyperopt configuration + :return: configuration as dictionary + """ + # If -i/--ticker-interval is used we override the configuration parameter + # (that will override the strategy configuration) + if 'ticker_interval' in self.args and self.args.ticker_interval: + config.update({'ticker_interval': self.args.ticker_interval}) + self.logger.info('Parameter -i/--ticker-interval detected ...') + self.logger.info('Using ticker_interval: %d ...', config.get('ticker_interval')) + + # If -l/--live is used we add it to the configuration + if 'live' in self.args and self.args.live: + config.update({'live': True}) + self.logger.info('Parameter -l/--live detected ...') + + # If --realistic-simulation is used we add it to the configuration + if 'realistic_simulation' in self.args and self.args.realistic_simulation: + config.update({'realistic_simulation': True}) + self.logger.info('Parameter --realistic-simulation detected ...') + self.logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) + + # If --timerange is used we add it to the configuration + if 'timerange' in self.args and self.args.timerange: + config.update({'timerange': self.args.timerange}) + self.logger.info('Parameter --timerange detected: %s ...', self.args.timerange) + + # If --datadir is used we add it to the configuration + if 'datadir' in self.args and self.args.datadir: + config.update({'datadir': self.args.datadir}) + self.logger.info('Parameter --datadir detected: %s ...', self.args.datadir) + + # If -r/--refresh-pairs-cached is used we add it to the configuration + if 'refresh_pairs' in self.args and self.args.refresh_pairs: + config.update({'refresh_pairs': True}) + self.logger.info('Parameter -r/--refresh-pairs-cached detected ...') + + # If --export is used we add it to the configuration + if 'export' in self.args and self.args.export: + config.update({'export': self.args.export}) + self.logger.info('Parameter --export detected: %s ...', self.args.export) + + return config + def _validate_config(self, conf: Dict[str, Any]) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema diff --git a/freqtrade/logger.py b/freqtrade/logger.py index 4526ac11c..83bbbfd0d 100644 --- a/freqtrade/logger.py +++ b/freqtrade/logger.py @@ -20,21 +20,61 @@ class Logger(object): """ self.name = name self.level = level + self.logger = None + self._init_logger() - def _init_logger(self) -> logging: + def _init_logger(self) -> None: """ Setup the bot logger configuration - :return: logging object + :return: None """ logging.basicConfig( level=self.level, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', ) + self.logger = self.get_logger() + self.set_level(self.level) + def get_logger(self) -> logging.RootLogger: """ Return the logger instance to use for sending message :return: the logger instance """ return logging.getLogger(self.name) + + def set_name(self, name: str) -> logging.RootLogger: + """ + Set the name of the logger + :param name: Name of the logger + :return: None + """ + self.name = name + self.logger = self.get_logger() + return self.logger + + def set_level(self, level) -> None: + """ + Set the level of the logger + :param level: + :return: None + """ + self.level = level + self.logger.setLevel(self.level) + + def set_format(self, log_format: str, propagate: bool = False) -> None: + """ + Set a new logging format + :return: None + """ + handler = logging.StreamHandler() + + len_handlers = len(self.logger.handlers) + if len_handlers: + self.logger.removeHandler(handler) + + handler.setFormatter(logging.Formatter(log_format)) + self.logger.addHandler(handler) + + self.logger.propagate = propagate diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 52ea55853..e7312292e 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,18 +1,16 @@ # pragma pylint: disable=missing-docstring -import logging import json import os from typing import Optional, List, Dict -from pandas import DataFrame +import gzip from freqtrade.exchange import get_ticker_history -from freqtrade.analyze import populate_indicators, parse_ticker_dataframe from freqtrade import misc +from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf -import gzip -logger = logging.getLogger(__name__) +logger = Logger(name=__name__).get_logger() def trim_tickerlist(tickerlist, timerange): @@ -84,21 +82,13 @@ def load_data(datadir: str, ticker_interval: int, pairs: Optional[List[str]] = N return result -def tickerdata_to_dataframe(data): - preprocessed = preprocess(data) - return preprocessed - - -def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: - """Creates a dataframe and populates indicators for given ticker data""" - return {pair: populate_indicators(parse_ticker_dataframe(pair_data)) - for pair, pair_data in tickerdata.items()} - - def make_testdata_path(datadir: str) -> str: """Return the path where testdata files are stored""" - return datadir or os.path.abspath(os.path.join(os.path.dirname(__file__), - '..', 'tests', 'testdata')) + return datadir or os.path.abspath( + os.path.join( + os.path.dirname(__file__), '..', 'tests', 'testdata' + ) + ) def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: @@ -115,11 +105,6 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: return True -def file_dump_json(filename, data): - with open(filename, "wt") as fp: - json.dump(data, fp) - - # FIX: 20180110, suggest rename interval to tick_interval def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool: """ @@ -142,8 +127,8 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> )) if os.path.isfile(filename): - with open(filename, "rt") as fp: - data = json.load(fp) + with open(filename, "rt") as file: + data = json.load(file) logger.debug("Current Start: {}".format(data[1]['T'])) logger.debug("Current End: {}".format(data[-1:][0]['T'])) else: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d6d016aba..60b014872 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1,235 +1,321 @@ -# pragma pylint: disable=missing-docstring,W0212 +# pragma pylint: disable=missing-docstring, W0212, too-many-arguments +""" +This module contains the backtesting logic +""" + +from typing import Dict, Tuple, Any import logging -from typing import Dict, Tuple - import arrow from pandas import DataFrame, Series from tabulate import tabulate -import freqtrade.misc as misc import freqtrade.optimize as optimize -from freqtrade import exchange -from freqtrade.analyze import populate_buy_trend, populate_sell_trend +from freqtrade.arguments import Arguments from freqtrade.exchange import Bittrex -from freqtrade.main import should_sell +from freqtrade.configuration import Configuration +from freqtrade import exchange +from freqtrade.analyze import Analyze +from freqtrade.logger import Logger +from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade -from freqtrade.strategy.strategy import Strategy - -logger = logging.getLogger(__name__) -def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: +class Backtesting(object): """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - all_dates = Series([]) - for pair_data in data.values(): - all_dates = all_dates.append(pair_data['date']) - all_dates.sort_values(inplace=True) - return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + Backtesting class, this class contains all the logic to run a backtest + To run a backtest: + backtesting = Backtesting(config) + backtesting.start() + """ + def __init__(self, config: Dict[str, Any]) -> None: + self.logging = Logger(name=__name__) + self.logger = self.logging.get_logger() -def generate_text_table( - data: Dict[str, Dict], results: DataFrame, stake_currency, ticker_interval) -> str: - """ - Generates and returns a text table for the given backtest data and the results dataframe - :return: pretty printed table with tabulate as str - """ - floatfmt = ('s', 'd', '.2f', '.8f', '.1f') - tabular_data = [] - headers = ['pair', 'buy count', 'avg profit %', - 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] - for pair in data: - result = results[results.currency == pair] + self.config = config + self.analyze = None + self.ticker_interval = None + self.tickerdata_to_dataframe = None + self.populate_buy_trend = None + self.populate_sell_trend = None + self._init() + + def _init(self) -> None: + """ + Init objects required for backtesting + :return: None + """ + self.analyze = Analyze(self.config) + self.ticker_interval = self.analyze.strategy.ticker_interval + self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe + self.populate_buy_trend = self.analyze.populate_buy_trend + self.populate_sell_trend = self.analyze.populate_sell_trend + exchange._API = Bittrex({'key': '', 'secret': ''}) + + @staticmethod + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + all_dates = Series([]) + for pair_data in data.values(): + all_dates = all_dates.append(pair_data['date']) + all_dates.sort_values(inplace=True) + return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + + def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: + """ + Generates and returns a text table for the given backtest data and the results dataframe + :return: pretty printed table with tabulate as str + """ + stake_currency = self.config.get('stake_currency') + ticker_interval = self.ticker_interval + + floatfmt = ('s', 'd', '.2f', '.8f', '.1f') + tabular_data = [] + headers = ['pair', 'buy count', 'avg profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for pair in data: + result = results[results.currency == pair] + tabular_data.append([ + pair, + len(result.index), + result.profit_percent.mean() * 100.0, + result.profit_BTC.sum(), + result.duration.mean() * ticker_interval, + len(result[result.profit_BTC > 0]), + len(result[result.profit_BTC < 0]) + ]) + + # Append Total tabular_data.append([ - pair, - len(result.index), - result.profit_percent.mean() * 100.0, - result.profit_BTC.sum(), - result.duration.mean() * ticker_interval, - len(result[result.profit_BTC > 0]), - len(result[result.profit_BTC < 0]) + 'TOTAL', + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.duration.mean() * ticker_interval, + len(results[results.profit_BTC > 0]), + len(results[results.profit_BTC < 0]) ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) - # Append Total - tabular_data.append([ - 'TOTAL', - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.duration.mean() * ticker_interval, - len(results[results.profit_BTC > 0]), - len(results[results.profit_BTC < 0]) - ]) - return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) + def _get_sell_trade_entry(self, pair, row, buy_subset, ticker, trade_count_lock, args): + stake_amount = args['stake_amount'] + max_open_trades = args.get('max_open_trades', 0) + trade = Trade( + open_rate=row.close, + open_date=row.date, + stake_amount=stake_amount, + amount=stake_amount / row.open, + fee=exchange.get_fee() + ) - -def get_sell_trade_entry(pair, row, buy_subset, ticker, trade_count_lock, args): - stake_amount = args['stake_amount'] - max_open_trades = args.get('max_open_trades', 0) - trade = Trade(open_rate=row.close, - open_date=row.date, - stake_amount=stake_amount, - amount=stake_amount / row.open, - fee=exchange.get_fee() - ) - - # calculate win/lose forwards from buy point - sell_subset = ticker[ticker.date > row.date][['close', 'date', 'sell']] - for row2 in sell_subset.itertuples(index=True): - if max_open_trades > 0: - # Increase trade_count_lock for every iteration - trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 - - # Buy is on is in the buy_subset there is a row that matches the date - # of the sell event - buy_signal = not buy_subset[buy_subset.date == row2.date].empty - if(should_sell(trade, row2.close, row2.date, buy_signal, row2.sell)): - return row2, (pair, - trade.calc_profit_percent(rate=row2.close), - trade.calc_profit(rate=row2.close), - row2.Index - row.Index - ), row2.date - return None - - -def backtest(args) -> DataFrame: - """ - Implements backtesting functionality - :param args: a dict containing: - stake_amount: btc amount to use for each trade - processed: a processed dictionary with format {pair, data} - max_open_trades: maximum number of concurrent trades (default: 0, disabled) - realistic: do we try to simulate realistic trades? (default: True) - sell_profit_only: sell if profit only - use_sell_signal: act on sell-signal - stoploss: use stoploss - :return: DataFrame - """ - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', True) - record = args.get('record', None) - records = [] - trades = [] - trade_count_lock: dict = {} - exchange._API = Bittrex({'key': '', 'secret': ''}) - for pair, pair_data in processed.items(): - pair_data['buy'], pair_data['sell'] = 0, 0 - ticker = populate_sell_trend(populate_buy_trend(pair_data)) - # for each buy point - lock_pair_until = None - headers = ['buy', 'open', 'close', 'date', 'sell'] - buy_subset = ticker[(ticker.buy == 1) & (ticker.sell == 0)][headers] - for row in buy_subset.itertuples(index=True): - if realistic: - if lock_pair_until is not None and row.date <= lock_pair_until: - continue + # calculate win/lose forwards from buy point + sell_subset = ticker[ticker.date > row.date][['close', 'date', 'sell']] + for row2 in sell_subset.itertuples(index=True): if max_open_trades > 0: - # Check if max_open_trades has already been reached for the given date - if not trade_count_lock.get(row.date, 0) < max_open_trades: - continue + # Increase trade_count_lock for every iteration + trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 - if max_open_trades > 0: - # Increase lock - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + # Buy is on is in the buy_subset there is a row that matches the date + # of the sell event + buy_signal = not buy_subset[buy_subset.date == row2.date].empty + if( + self.analyze.should_sell( + trade=trade, + rate=row2.close, + date=row2.date, + buy=buy_signal, + sell=row2.sell + ) + ): + return \ + row2, \ + ( + pair, + trade.calc_profit_percent(rate=row2.close), + trade.calc_profit(rate=row2.close), + row2.Index - row.Index + ),\ + row2.date + return None - ret = get_sell_trade_entry(pair, row, buy_subset, ticker, - trade_count_lock, args) - if ret: - row2, trade_entry, next_date = ret - lock_pair_until = next_date - trades.append(trade_entry) - if record: - # Note, need to be json.dump friendly - # record a tuple of pair, current_profit_percent, - # entry-date, duration - records.append((pair, trade_entry[1], - row.date.strftime('%s'), - row2.date.strftime('%s'), - row.Index, trade_entry[3])) - # For now export inside backtest(), maybe change so that backtest() - # returns a tuple like: (dataframe, records, logs, etc) - if record and record.find('trades') >= 0: - logger.info('Dumping backtest results') - misc.file_dump_json('backtest-result.json', records) - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] - return DataFrame.from_records(trades, columns=labels) + def backtest(self, args) -> DataFrame: + """ + Implements backtesting functionality + + NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. + Of course try to not have ugly code. By some accessor are sometime slower than functions. + Avoid, logging on this method + + :param args: a dict containing: + stake_amount: btc amount to use for each trade + processed: a processed dictionary with format {pair, data} + max_open_trades: maximum number of concurrent trades (default: 0, disabled) + realistic: do we try to simulate realistic trades? (default: True) + sell_profit_only: sell if profit only + use_sell_signal: act on sell-signal + stoploss: use stoploss + :return: DataFrame + """ + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', True) + record = args.get('record', None) + records = [] + trades = [] + trade_count_lock = {} + for pair, pair_data in processed.items(): + pair_data['buy'], pair_data['sell'] = 0, 0 + ticker = self.populate_sell_trend( + self.populate_buy_trend(pair_data) + ) + # for each buy point + lock_pair_until = None + headers = ['buy', 'open', 'close', 'date', 'sell'] + buy_subset = ticker[(ticker.buy == 1) & (ticker.sell == 0)][headers] + for row in buy_subset.itertuples(index=True): + if realistic: + if lock_pair_until is not None and row.date <= lock_pair_until: + continue + if max_open_trades > 0: + # Check if max_open_trades has already been reached for the given date + if not trade_count_lock.get(row.date, 0) < max_open_trades: + continue + + if max_open_trades > 0: + # Increase lock + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + + ret = self._get_sell_trade_entry( + pair=pair, + row=row, + buy_subset=buy_subset, + ticker=ticker, + trade_count_lock=trade_count_lock, + args=args + ) + + if ret: + row2, trade_entry, next_date = ret + lock_pair_until = next_date + trades.append(trade_entry) + if record: + # Note, need to be json.dump friendly + # record a tuple of pair, current_profit_percent, + # entry-date, duration + records.append((pair, trade_entry[1], + row.date.strftime('%s'), + row2.date.strftime('%s'), + row.Index, trade_entry[3])) + # For now export inside backtest(), maybe change so that backtest() + # returns a tuple like: (dataframe, records, logs, etc) + if record and record.find('trades') >= 0: + self.logger.info('Dumping backtest results') + file_dump_json('backtest-result.json', records) + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + return DataFrame.from_records(trades, columns=labels) + + def start(self) -> None: + """ + Run a backtesting end-to-end + :return: None + """ + data = {} + pairs = self.config['exchange']['pair_whitelist'] + + if self.config.get('live'): + self.logger.info('Downloading data for all pairs in whitelist ...') + for pair in pairs: + data[pair] = exchange.get_ticker_history(pair, self.ticker_interval) + else: + self.logger.info('Using local backtesting data (using whitelist in given config) ...') + self.logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + self.logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + timerange=timerange + ) + + max_open_trades = self.config.get('max_open_trades', 0) + + preprocessed = self.tickerdata_to_dataframe(data) + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + self.logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + # Execute backtest and print results + sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False) + use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False) + results = self.backtest( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'realistic': self.config.get('realistic_simulation', False), + 'sell_profit_only': sell_profit_only, + 'use_sell_signal': use_sell_signal, + 'stoploss': self.analyze.strategy.stoploss, + 'record': self.config.get('export') + } + ) + + self.logging.set_format('%(message)s') + self.logger.info( + '\n==================================== ' + 'BACKTESTING REPORT' + ' ====================================\n' + '%s', + self._generate_text_table( + data, + results + ) + ) -def start(args): +def setup_configuration(args) -> Dict[str, Any]: + """ + Prepare the configuration for the backtesting + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + return config + + +def start(args) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + # Initialize logger - logging.basicConfig( - level=args.loglevel, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) + logger = Logger(name=__name__).get_logger() + logger.info('Starting freqtrade in Backtesting mode') - exchange._API = Bittrex({'key': '', 'secret': ''}) + # Initialize configuration + config = setup_configuration(args) - logger.info('Using config: %s ...', args.config) - config = misc.load_config(args.config) - - # If -i/--ticker-interval is use we override the configuration parameter - # (that will override the strategy configuration) - if args.ticker_interval: - config.update({'ticker_interval': args.ticker_interval}) - - # init the strategy to use - config.update({'strategy': args.strategy}) - strategy = Strategy() - strategy.init(config) - - logger.info('Using ticker_interval: %d ...', strategy.ticker_interval) - - data = {} - pairs = config['exchange']['pair_whitelist'] - if args.live: - logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = exchange.get_ticker_history(pair, strategy.ticker_interval) - else: - logger.info('Using local backtesting data (using whitelist in given config) ...') - logger.info('Using stake_currency: %s ...', config['stake_currency']) - logger.info('Using stake_amount: %s ...', config['stake_amount']) - - timerange = misc.parse_timerange(args.timerange) - data = optimize.load_data(args.datadir, - pairs=pairs, - ticker_interval=strategy.ticker_interval, - refresh_pairs=args.refresh_pairs, - timerange=timerange) - max_open_trades = 0 - if args.realistic_simulation: - logger.info('Using max_open_trades: %s ...', config['max_open_trades']) - max_open_trades = config['max_open_trades'] - - # Monkey patch config - from freqtrade import main - main._CONF = config - - preprocessed = optimize.tickerdata_to_dataframe(data) - # Print timeframe - min_date, max_date = get_timeframe(preprocessed) - logger.info('Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date-min_date).days) - # Execute backtest and print results - sell_profit_only = config.get('experimental', {}).get('sell_profit_only', False) - use_sell_signal = config.get('experimental', {}).get('use_sell_signal', False) - results = backtest({'stake_amount': config['stake_amount'], - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'realistic': args.realistic_simulation, - 'sell_profit_only': sell_profit_only, - 'use_sell_signal': use_sell_signal, - 'stoploss': strategy.stoploss, - 'record': args.export - }) - logger.info( - '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa - generate_text_table(data, results, config['stake_currency'], strategy.ticker_interval) - ) + # Initialize backtesting object + backtesting = Backtesting(config) + backtesting.start() diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 26a61e923..d101e6e9a 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -269,13 +269,6 @@ def result(): return Analyze.parse_ticker_dataframe(json.load(data_file)) -@pytest.fixture -def default_strategy(): - strategy = Strategy() - strategy.init({'strategy': 'default_strategy'}) - return strategy - - # FIX: # Create an fixture/function # that inserts a trade of some type and open-status diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 0dd4f777a..7a0158162 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,14 +1,24 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103 +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument -import logging +import json import math +from typing import List +from copy import deepcopy from unittest.mock import MagicMock import pandas as pd -from freqtrade import exchange, optimize -from freqtrade.exchange import Bittrex -from freqtrade.optimize import preprocess -from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe -import freqtrade.optimize.backtesting as backtesting +from freqtrade import optimize +from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration +from freqtrade.arguments import Arguments +from freqtrade.analyze import Analyze +import freqtrade.tests.conftest as tt # test tools + + +# Avoid to reinit the same object again and again +_BACKTESTING = Backtesting(tt.default_conf()) + + +def get_args(args) -> List[str]: + return Arguments(args, '').get_parsed_arg() def trim_dictlist(dict_list, num): @@ -18,60 +28,6 @@ def trim_dictlist(dict_list, num): return new -def test_generate_text_table(): - results = pd.DataFrame( - { - 'currency': ['BTC_ETH', 'BTC_ETH'], - 'profit_percent': [0.1, 0.2], - 'profit_BTC': [0.2, 0.4], - 'duration': [10, 30], - 'profit': [2, 0], - 'loss': [0, 0] - } - ) - print(generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5)) - assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == ( - 'pair buy count avg profit % total profit BTC avg duration profit loss\n' # noqa - '------- ----------- -------------- ------------------ -------------- -------- ------\n' # noqa - 'BTC_ETH 2 15.00 0.60000000 100.0 2 0\n' # noqa - 'TOTAL 2 15.00 0.60000000 100.0 2 0') # noqa - - -def test_get_timeframe(default_strategy): - data = preprocess(optimize.load_data( - None, ticker_interval=1, pairs=['BTC_UNITEST'])) - min_date, max_date = get_timeframe(data) - assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' - assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' - - -def test_backtest(default_strategy, default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) - data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 10, - 'realistic': True}) - assert not results.empty - - -def test_backtest_1min_ticker_interval(default_strategy, default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - # Run a backtesting for an exiting 5min ticker_interval - data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST']) - data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 1, - 'realistic': True}) - assert not results.empty - - def load_data_test(what): timerange = ((None, 'line'), None, -100) data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'], timerange=timerange) @@ -115,79 +71,324 @@ def load_data_test(what): return data -def simple_backtest(config, contour, num_results): +def simple_backtest(config, contour, num_results) -> None: + backtesting = _BACKTESTING + data = load_data_test(contour) - processed = optimize.preprocess(data) + processed = backtesting.tickerdata_to_dataframe(data) assert isinstance(processed, dict) - results = backtest({'stake_amount': config['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'realistic': True}) + results = backtesting.backtest( + { + 'stake_amount': config['stake_amount'], + 'processed': processed, + 'max_open_trades': 1, + 'realistic': True + } + ) # results :: assert len(results) == num_results -# Test backtest on offline data -# loaded by freqdata/optimize/__init__.py::load_data() - - -def test_backtest2(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) - data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 10, - 'realistic': True}) - assert not results.empty - - -def test_processed(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - dict_of_tickerrows = load_data_test('raise') - dataframes = optimize.preprocess(dict_of_tickerrows) - dataframe = dataframes['BTC_UNITEST'] - cols = dataframe.columns - # assert the dataframe got some of the indicator columns - for col in ['close', 'high', 'low', 'open', 'date', - 'ema50', 'ao', 'macd', 'plus_dm']: - assert col in cols - - -def test_backtest_pricecontours(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - tests = [['raise', 17], ['lower', 0], ['sine', 17]] - for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres) - - def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False, timerange=None): tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1, timerange=timerange) pairdata = {'BTC_UNITEST': tickerdata} return pairdata -def test_backtest_start(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - default_conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.misc.load_config', new=lambda s: default_conf) - mocker.patch.multiple('freqtrade.optimize', - load_data=mocked_load_data) - args = MagicMock() - args.ticker_interval = 1 - args.level = 10 - args.live = False - args.datadir = None - args.export = None - args.timerange = '-100' # needed due to MagicMock malleability - backtesting.start(args) +# Unit tests +def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'backtesting' + ] + + config = setup_configuration(get_args(args)) + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert tt.log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert not tt.log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + + assert 'live' not in config + assert not tt.log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation' not in config + assert not tt.log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + + assert 'refresh_pairs' not in config + assert not tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + + assert 'timerange' not in config + assert 'export' not in config + + +def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + '--datadir', '/foo/bar', + 'backtesting', + '--ticker-interval', '1', + '--live', + '--realistic-simulation', + '--refresh-pairs-cached', + '--timerange', ':100', + '--export', '/bar/foo' + ] + + config = setup_configuration(get_args(args)) + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert tt.log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert tt.log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + assert tt.log_has( + 'Using ticker_interval: 1 ...', + caplog.record_tuples + ) + + assert 'live' in config + assert tt.log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation'in config + assert tt.log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert tt.log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'refresh_pairs'in config + assert tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + + import pprint + pprint.pprint(caplog.record_tuples) + pprint.pprint(config['timerange']) + assert 'timerange' in config + assert tt.log_has( + 'Parameter --timerange detected: {} ...'.format(config['timerange']), + caplog.record_tuples + ) + + assert 'export' in config + assert tt.log_has( + 'Parameter --export detected: {} ...'.format(config['export']), + caplog.record_tuples + ) + + +def test_start(mocker, default_conf, caplog) -> None: + """ + Test start() function + """ + start_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting.start', start_mock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'backtesting' + ] + args = get_args(args) + start(args) + assert tt.log_has( + 'Starting freqtrade in Backtesting mode', + caplog.record_tuples + ) + assert start_mock.call_count == 1 + + +def test_backtesting__init__(mocker, default_conf) -> None: + """ + Test Backtesting.__init__() method + """ + init_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting._init', init_mock) + + backtesting = Backtesting(default_conf) + assert backtesting.config == default_conf + assert backtesting.analyze is None + assert backtesting.ticker_interval is None + assert backtesting.tickerdata_to_dataframe is None + assert backtesting.populate_buy_trend is None + assert backtesting.populate_sell_trend is None + assert init_mock.call_count == 1 + + +def test_backtesting_init(default_conf) -> None: + """ + Test Backtesting._init() method + """ + backtesting = Backtesting(default_conf) + assert backtesting.config == default_conf + assert isinstance(backtesting.analyze, Analyze) + assert backtesting.ticker_interval == 5 + assert callable(backtesting.tickerdata_to_dataframe) + assert callable(backtesting.populate_buy_trend) + assert callable(backtesting.populate_sell_trend) + + +def test_get_timeframe() -> None: + """ + Test Backtesting.get_timeframe() method + """ + backtesting = _BACKTESTING + + data = backtesting.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval=1, + pairs=['BTC_UNITEST'] + ) + ) + min_date, max_date = backtesting.get_timeframe(data) + assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' + assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' + + +def test_generate_text_table(): + """ + Test Backtesting.generate_text_table() method + """ + backtesting = _BACKTESTING + + results = pd.DataFrame( + { + 'currency': ['BTC_ETH', 'BTC_ETH'], + 'profit_percent': [0.1, 0.2], + 'profit_BTC': [0.2, 0.4], + 'duration': [10, 30], + 'profit': [2, 0], + 'loss': [0, 0] + } + ) + + result_str = ( + 'pair buy count avg profit % ' + 'total profit BTC avg duration profit loss\n' + '------- ----------- -------------- ' + '------------------ -------------- -------- ------\n' + 'BTC_ETH 2 15.00 ' + '0.60000000 100.0 2 0\n' + 'TOTAL 2 15.00 ' + '0.60000000 100.0 2 0' + ) + + assert backtesting._generate_text_table(data={'BTC_ETH': {}}, results=results) == result_str + + +def test_backtesting_start(default_conf, mocker, caplog) -> None: + """ + Test Backtesting.start() method + """ + mocker.patch.multiple('freqtrade.optimize', load_data=mocked_load_data) + mocker.patch('freqtrade.exchange.get_ticker_history', MagicMock) + + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] + conf['ticker_interval'] = 1 + conf['live'] = False + conf['datadir'] = None + conf['export'] = None + conf['timerange'] = '-100' + + backtesting = Backtesting(conf) + backtesting.start() # check the logs, that will contain the backtest result - exists = ['Using max_open_trades: 1 ...', - 'Using stake_amount: 0.001 ...', - 'Measuring data from 2017-11-14T21:17:00+00:00 ' - 'up to 2017-11-14T22:59:00+00:00 (0 days)..'] + exists = [ + 'Using local backtesting data (using whitelist in given config) ...', + 'Using stake_currency: BTC ...', + 'Using stake_amount: 0.001 ...', + 'Measuring data from 2017-11-14T21:17:00+00:00 ' + 'up to 2017-11-14T22:59:00+00:00 (0 days)..' + ] for line in exists: - assert ('freqtrade.optimize.backtesting', - logging.INFO, - line) in caplog.record_tuples + assert tt.log_has(line, caplog.record_tuples) + + +def test_backtest(default_conf) -> None: + """ + Test Backtesting.backtest() method + """ + backtesting = _BACKTESTING + + data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) + data = trim_dictlist(data, -200) + results = backtesting.backtest( + { + 'stake_amount': default_conf['stake_amount'], + 'processed': backtesting.tickerdata_to_dataframe(data), + 'max_open_trades': 10, + 'realistic': True + } + ) + assert not results.empty + + +def test_backtest_1min_ticker_interval(default_conf) -> None: + """ + Test Backtesting.backtest() method with 1 min ticker + """ + backtesting = _BACKTESTING + + # Run a backtesting for an exiting 5min ticker_interval + data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST']) + data = trim_dictlist(data, -200) + results = backtesting.backtest( + { + 'stake_amount': default_conf['stake_amount'], + 'processed': backtesting.tickerdata_to_dataframe(data), + 'max_open_trades': 1, + 'realistic': True + } + ) + assert not results.empty + + +def test_processed() -> None: + """ + Test Backtesting.backtest() method with offline data + """ + backtesting = _BACKTESTING + + dict_of_tickerrows = load_data_test('raise') + dataframes = backtesting.tickerdata_to_dataframe(dict_of_tickerrows) + dataframe = dataframes['BTC_UNITEST'] + cols = dataframe.columns + # assert the dataframe got some of the indicator columns + for col in ['close', 'high', 'low', 'open', 'date', + 'ema50', 'ao', 'macd', 'plus_dm']: + assert col in cols + + +def test_backtest_pricecontours(default_conf) -> None: + tests = [['raise', 17], ['lower', 0], ['sine', 17]] + for [contour, numres] in tests: + simple_backtest(default_conf, contour, numres) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 8241d3822..b1459198d 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -5,10 +5,11 @@ import json import logging import uuid from shutil import copyfile -from freqtrade import exchange, optimize -from freqtrade.exchange import Bittrex +from freqtrade import optimize +from freqtrade.analyze import Analyze from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ - download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, file_dump_json + download_backtesting_testdata, load_tickerdata_file, trim_tickerlist +from freqtrade.misc import file_dump_json # Change this if modifying BTC_UNITEST testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -45,12 +46,11 @@ def _clean_test_file(file: str) -> None: os.rename(file_swp, file) -def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_30min_ticker(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 30 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_UNITTEST-30.json' _backup_file(file, copy_file=True) @@ -62,12 +62,11 @@ def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog): _clean_test_file(file) -def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_5min_ticker(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 5 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_ETH-5.json' _backup_file(file, copy_file=True) @@ -79,12 +78,11 @@ def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog): _clean_test_file(file) -def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 1 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_ETH-1.json' _backup_file(file, copy_file=True) @@ -96,12 +94,11 @@ def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog): _clean_test_file(file) -def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 1 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_MEME-1.json' _backup_file(file) @@ -113,14 +110,12 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl _clean_test_file(file) -def test_testdata_path(): +def test_testdata_path() -> None: assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) -def test_download_pairs(default_conf, ticker_history, mocker): +def test_download_pairs(ticker_history, mocker) -> None: mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' @@ -157,13 +152,10 @@ def test_download_pairs(default_conf, ticker_history, mocker): _clean_test_file(file2_5) -def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_download_pairs_exception(ticker_history, mocker, caplog) -> None: mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' @@ -179,10 +171,8 @@ def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog): 'Failed to download the pair: "BTC-MEME", Interval: 1 min') in caplog.record_tuples -def test_download_backtesting_testdata(default_conf, ticker_history, mocker): +def test_download_backtesting_testdata(ticker_history, mocker) -> None: mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) # Download a 1 min ticker file file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' @@ -200,7 +190,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): _clean_test_file(file2) -def test_download_backtesting_testdata2(mocker): +def test_download_backtesting_testdata2(mocker) -> None: tick = [{'T': 'bar'}, {'T': 'foo'}] mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) @@ -208,7 +198,7 @@ def test_download_backtesting_testdata2(mocker): assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) -def test_load_tickerdata_file(): +def test_load_tickerdata_file() -> None: # 7 does not exist in either format. assert not load_tickerdata_file(None, 'BTC_UNITEST', 7) # 1 exists only as a .json @@ -219,22 +209,28 @@ def test_load_tickerdata_file(): assert _BTC_UNITTEST_LENGTH == len(tickerdata) -def test_init(default_conf, mocker): +def test_init(default_conf, mocker) -> None: conf = {'exchange': {'pair_whitelist': []}} mocker.patch('freqtrade.optimize.hyperopt_optimize_conf', return_value=conf) - assert {} == optimize.load_data('', pairs=[], refresh_pairs=True, - ticker_interval=int(default_conf['ticker_interval'])) + assert {} == optimize.load_data( + '', + pairs=[], + refresh_pairs=True, + ticker_interval=int(default_conf['ticker_interval']) + ) -def test_tickerdata_to_dataframe(): +def test_tickerdata_to_dataframe(default_conf) -> None: + analyze = Analyze(default_conf) + timerange = ((None, 'line'), None, -100) tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) tickerlist = {'BTC_UNITEST': tick} - data = optimize.tickerdata_to_dataframe(tickerlist) + data = analyze.tickerdata_to_dataframe(tickerlist) assert len(data['BTC_UNITEST']) == 100 -def test_trim_tickerlist(): +def test_trim_tickerlist() -> None: with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: ticker_list = json.load(data_file) ticker_list_len = len(ticker_list) @@ -279,8 +275,11 @@ def test_trim_tickerlist(): assert ticker_list_len == ticker_len -def test_file_dump_json(): - +def test_file_dump_json() -> None: + """ + Test file_dump_json() + :return: None + """ file = 'freqtrade/tests/testdata/test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'} diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index c9def6a9e..9d5970f25 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -1,5 +1,4 @@ -# pragma pylint: disable=protected-access, invalid-name, missing-docstring - +# pragma pylint: disable=protected-access, invalid-name """ Unit test file for configuration.py """ @@ -18,7 +17,6 @@ import freqtrade.tests.conftest as tt # test tools def test_configuration_object() -> None: """ Test the Constants object has the mandatory Constants - :return: None """ assert hasattr(Configuration, '_load_config') assert hasattr(Configuration, '_load_config_file') @@ -30,12 +28,11 @@ def test_configuration_object() -> None: def test_load_config_invalid_pair(default_conf, mocker) -> None: """ Test the configuration validator with an invalid PAIR format - :param default_conf: Configuration already read from a file (JSON format) - :return: None """ mocker.patch.multiple( 'freqtrade.configuration.Configuration', - _load_config=MagicMock(return_value=[]) + _load_config=MagicMock(return_value=[]), + show_info=MagicMock ) conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'].append('BTC-ETH') @@ -48,12 +45,11 @@ def test_load_config_invalid_pair(default_conf, mocker) -> None: def test_load_config_missing_attributes(default_conf, mocker) -> None: """ Test the configuration validator with a missing attribute - :param default_conf: Configuration already read from a file (JSON format) - :return: None """ mocker.patch.multiple( 'freqtrade.configuration.Configuration', - _load_config=MagicMock(return_value=[]) + _load_config=MagicMock(return_value=[]), + show_info=MagicMock ) conf = deepcopy(default_conf) conf.pop('exchange') @@ -65,12 +61,12 @@ def test_load_config_missing_attributes(default_conf, mocker) -> None: def test_load_config_file(default_conf, mocker, caplog) -> None: """ - Test _load_config_file() method - :return: + Test Configuration._load_config_file() method """ mocker.patch.multiple( 'freqtrade.configuration.Configuration', - _load_config=MagicMock(return_value=[]) + _load_config=MagicMock(return_value=[]), + show_info=MagicMock ) file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) @@ -85,6 +81,9 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: def test_load_config(default_conf, mocker) -> None: + """ + Test Configuration._load_config() without any cli params + """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -100,6 +99,9 @@ def test_load_config(default_conf, mocker) -> None: def test_load_config_with_params(default_conf, mocker) -> None: + """ + Test Configuration._load_config() with cli params used + """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -123,6 +125,9 @@ def test_load_config_with_params(default_conf, mocker) -> None: def test_show_info(default_conf, mocker, caplog) -> None: + """ + Test Configuration.show_info() + """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -155,3 +160,118 @@ def test_show_info(default_conf, mocker, caplog) -> None: 'Dry run is disabled. (--dry_run_db ignored)', caplog.record_tuples ) + + +def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.Configuration.show_info', MagicMock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'backtesting' + ] + + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert tt.log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert not tt.log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + + assert 'live' not in config + assert not tt.log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation' not in config + assert not tt.log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + + assert 'refresh_pairs' not in config + assert not tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + + assert 'timerange' not in config + assert 'export' not in config + + +def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.Configuration.show_info', MagicMock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + '--datadir', '/foo/bar', + 'backtesting', + '--ticker-interval', '1', + '--live', + '--realistic-simulation', + '--refresh-pairs-cached', + '--timerange', ':100', + '--export', '/bar/foo' + ] + + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert tt.log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert tt.log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + assert tt.log_has( + 'Using ticker_interval: 1 ...', + caplog.record_tuples + ) + + assert 'live' in config + assert tt.log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation'in config + assert tt.log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert tt.log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'refresh_pairs'in config + assert tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + + import pprint + pprint.pprint(caplog.record_tuples) + pprint.pprint(config['timerange']) + assert 'timerange' in config + assert tt.log_has( + 'Parameter --timerange detected: {} ...'.format(config['timerange']), + caplog.record_tuples + ) + + assert 'export' in config + assert tt.log_has( + 'Parameter --export detected: {} ...'.format(config['export']), + caplog.record_tuples + ) + diff --git a/freqtrade/tests/test_logger.py b/freqtrade/tests/test_logger.py index e492ce097..d6e7a2e06 100644 --- a/freqtrade/tests/test_logger.py +++ b/freqtrade/tests/test_logger.py @@ -26,11 +26,12 @@ def test_logger_object() -> None: def test_get_logger() -> None: """ - Test logger.get_logger() + Test Logger.get_logger() and Logger._init_logger() :return: None """ - logger = Logger(name='Foo', level=logging.WARNING) + logger = Logger(name='get_logger', level=logging.WARNING) get_logger = logger.get_logger() + assert logger.logger is get_logger assert get_logger is not None assert hasattr(get_logger, 'debug') assert hasattr(get_logger, 'info') @@ -39,15 +40,57 @@ def test_get_logger() -> None: assert hasattr(get_logger, 'exception') +def test_set_name() -> None: + """ + Test Logger.set_name() + :return: None + """ + logger = Logger(name='set_name') + assert logger.name == 'set_name' + + logger.set_name('set_name_new') + assert logger.name == 'set_name_new' + + +def test_set_level() -> None: + """ + Test Logger.set_name() + :return: None + """ + logger = Logger(name='Foo', level=logging.WARNING) + assert logger.level == logging.WARNING + assert logger.get_logger().level == logging.WARNING + + logger.set_level(logging.INFO) + assert logger.level == logging.INFO + assert logger.get_logger().level == logging.INFO + + def test_sending_msg(caplog) -> None: """ Test send a logging message :return: None """ - logger = Logger(name='FooBar', level=logging.WARNING).get_logger() + logger = Logger(name='sending_msg', level=logging.WARNING).get_logger() logger.info('I am an INFO message') - assert('FooBar', logging.INFO, 'I am an INFO message') not in caplog.record_tuples + assert('sending_msg', logging.INFO, 'I am an INFO message') not in caplog.record_tuples logger.warning('I am an WARNING message') - assert ('FooBar', logging.WARNING, 'I am an WARNING message') in caplog.record_tuples + assert ('sending_msg', logging.WARNING, 'I am an WARNING message') in caplog.record_tuples + + +def test_set_format(caplog) -> None: + """ + Test Logger.set_format() + :return: None + """ + log = Logger(name='set_format') + logger = log.get_logger() + + logger.info('I am the first message') + assert ('set_format', logging.INFO, 'I am the first message') in caplog.record_tuples + + log.set_format(log_format='%(message)s', propagate=True) + logger.info('I am the second message') + assert ('set_format', logging.INFO, 'I am the second message') in caplog.record_tuples From 383fb6d20ec7934f518df67cf2edcbaaa7e45786 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 11 Feb 2018 22:10:21 -0800 Subject: [PATCH 10/56] Add a class Arguments to manage cli arguments passed to the bot --- freqtrade/arguments.py | 247 ++++++++++++++++++++++++++++++ freqtrade/tests/test_arguments.py | 127 +++++++++++++++ 2 files changed, 374 insertions(+) create mode 100644 freqtrade/arguments.py create mode 100644 freqtrade/tests/test_arguments.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py new file mode 100644 index 000000000..0fa412c85 --- /dev/null +++ b/freqtrade/arguments.py @@ -0,0 +1,247 @@ +""" +This module contains the argument manager class +""" + +import argparse +import os +import re +import logging +from typing import List + +from freqtrade import __version__ +from freqtrade.constants import Constants + + +class Arguments(object): + """ + Arguments Class. Manage the arguments received by the cli + """ + + def __init__(self, args: List[str], description: str): + self.args = args + self.parsed_arg = None + self.parser = argparse.ArgumentParser(description=description) + self._common_args_parser() + self._build_subcommands() + + def get_parsed_arg(self) -> List[str]: + """ + Return the list of arguments + :return: List[str] List of arguments + """ + self.parsed_arg = self._parse_args() + + return self.parsed_arg + + def _parse_args(self) -> List[str]: + """ + Parses given arguments and returns an argparse Namespace instance. + """ + parsed_arg = self.parser.parse_args(self.args) + + return parsed_arg + + def _common_args_parser(self) -> None: + """ + Parses given common arguments and returns them as a parsed object. + """ + self.parser.add_argument( + '-v', '--verbose', + help='be verbose', + action='store_const', + dest='loglevel', + const=logging.DEBUG, + default=logging.INFO, + ) + self.parser.add_argument( + '--version', + action='version', + version='%(prog)s {}'.format(__version__), + ) + self.parser.add_argument( + '-c', '--config', + help='specify configuration file (default: config.json)', + dest='config', + default='config.json', + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '--datadir', + help='path to backtest data (default freqdata/tests/testdata)', + dest='datadir', + default=os.path.join('freqtrade', 'tests', 'testdata'), + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '-s', '--strategy', + help='specify strategy file (default: freqtrade/strategy/default_strategy.py)', + dest='strategy', + default='default_strategy', + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '--dynamic-whitelist', + help='dynamically generate and update whitelist \ + based on 24h BaseVolume (Default 20 currencies)', # noqa + dest='dynamic_whitelist', + const=Constants.DYNAMIC_WHITELIST, + type=int, + metavar='INT', + nargs='?', + ) + self.parser.add_argument( + '--dry-run-db', + help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ + instead of memory DB. Work only if dry_run is enabled.', + action='store_true', + dest='dry_run_db', + ) + + @staticmethod + def _backtesting_options(parser: argparse.ArgumentParser) -> None: + """ + Parses given arguments for Backtesting scripts. + """ + parser.add_argument( + '-l', '--live', + action='store_true', + dest='live', + help='using live data', + ) + parser.add_argument( + '-i', '--ticker-interval', + help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', + dest='ticker_interval', + type=int, + metavar='INT', + ) + parser.add_argument( + '--realistic-simulation', + help='uses max_open_trades from config to simulate real world limitations', + action='store_true', + dest='realistic_simulation', + ) + parser.add_argument( + '-r', '--refresh-pairs-cached', + help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \ + Use it if you want to run your backtesting with up-to-date data.', + action='store_true', + dest='refresh_pairs', + ) + parser.add_argument( + '--export', + help='Export backtest results, argument are: trades\ + Example --export=trades', + type=str, + default=None, + dest='export', + ) + parser.add_argument( + '--timerange', + help='Specify what timerange of data to use.', + default=None, + type=str, + dest='timerange', + ) + + @staticmethod + def _hyperopt_options(parser: argparse.ArgumentParser) -> None: + """ + Parses given arguments for Hyperopt scripts. + """ + parser.add_argument( + '-e', '--epochs', + help='specify number of epochs (default: 100)', + dest='epochs', + default=Constants.HYPEROPT_EPOCH, + type=int, + metavar='INT', + ) + parser.add_argument( + '--use-mongodb', + help='parallelize evaluations with mongodb (requires mongod in PATH)', + dest='mongodb', + action='store_true', + ) + parser.add_argument( + '-i', '--ticker-interval', + help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', + dest='ticker_interval', + type=int, + metavar='INT', + ) + parser.add_argument( + '--timerange', + help='Specify what timerange of data to use.', + default=None, + type=str, + dest='timerange', + ) + + def _build_subcommands(self) -> None: + """ + Builds and attaches all subcommands + :return: None + """ + from freqtrade.optimize import backtesting, hyperopt + + subparsers = self.parser.add_subparsers(dest='subparser') + + # Add backtesting subcommand + backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') + backtesting_cmd.set_defaults(func=backtesting.start) + self._backtesting_options(backtesting_cmd) + + # Add hyperopt subcommand + hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') + hyperopt_cmd.set_defaults(func=hyperopt.start) + self._hyperopt_options(hyperopt_cmd) + + @staticmethod + def parse_timerange(text: str) -> (List, int, int): + """ + Parse the value of the argument --timerange to determine what is the range desired + :param text: value from --timerange + :return: Start and End range period + """ + if text is None: + return None + syntax = [(r'^-(\d{8})$', (None, 'date')), + (r'^(\d{8})-$', ('date', None)), + (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^(-\d+)$', (None, 'line')), + (r'^(\d+)-$', ('line', None)), + (r'^(\d+)-(\d+)$', ('index', 'index'))] + for rex, stype in syntax: + # Apply the regular expression to text + match = re.match(rex, text) + if match: # Regex has matched + rvals = match.groups() + index = 0 + start = None + stop = None + if stype[0]: + start = rvals[index] + if stype[0] != 'date': + start = int(start) + index += 1 + if stype[1]: + stop = rvals[index] + if stype[1] != 'date': + stop = int(stop) + return (stype, start, stop) + raise Exception('Incorrect syntax for timerange "%s"' % text) + + def scripts_options(self): + """ + Parses given arguments for plot scripts. + """ + self.parser.add_argument( + '-p', '--pair', + help='Show profits for only this pairs. Pairs are comma-separated.', + dest='pair', + default=None + ) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py new file mode 100644 index 000000000..08125cfdc --- /dev/null +++ b/freqtrade/tests/test_arguments.py @@ -0,0 +1,127 @@ +# pragma pylint: disable=missing-docstring, C0103 + +""" +Unit test file for arguments.py +""" + +import argparse +import logging +import pytest + +from freqtrade.arguments import Arguments + + +def test_arguments_object() -> None: + """ + Test the Arguments object has the mandatory methods + :return: None + """ + assert hasattr(Arguments, 'get_parsed_arg') + assert hasattr(Arguments, '_parse_args') + assert hasattr(Arguments, 'parse_timerange') + assert hasattr(Arguments, 'scripts_options') + + +# Parse common command-line-arguments. Used for all tools +def test_parse_args_none() -> None: + arguments = Arguments([], '') + assert isinstance(arguments, Arguments) + assert isinstance(arguments.parser, argparse.ArgumentParser) + assert isinstance(arguments.parser, argparse.ArgumentParser) + + +def test_parse_args_defaults() -> None: + args = Arguments([], '').get_parsed_arg() + assert args.config == 'config.json' + assert args.dynamic_whitelist is None + assert args.loglevel == logging.INFO + + +def test_parse_args_config() -> None: + args = Arguments(['-c', '/dev/null'], '').get_parsed_arg() + assert args.config == '/dev/null' + + args = Arguments(['--config', '/dev/null'], '').get_parsed_arg() + assert args.config == '/dev/null' + + +def test_parse_args_verbose() -> None: + args = Arguments(['-v'], '').get_parsed_arg() + assert args.loglevel == logging.DEBUG + + args = Arguments(['--verbose'], '').get_parsed_arg() + assert args.loglevel == logging.DEBUG + + +def test_scripts_options() -> None: + arguments = Arguments(['-p', 'BTC_ETH'], '') + arguments.scripts_options() + args = arguments.get_parsed_arg() + assert args.pair == 'BTC_ETH' + + +def test_parse_args_version() -> None: + with pytest.raises(SystemExit, match=r'0'): + Arguments(['--version'], '').get_parsed_arg() + + +def test_parse_args_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['-c'], '').get_parsed_arg() + + +def test_parse_args_dynamic_whitelist() -> None: + args = Arguments(['--dynamic-whitelist'], '').get_parsed_arg() + assert args.dynamic_whitelist == 20 + + +def test_parse_args_dynamic_whitelist_10() -> None: + args = Arguments(['--dynamic-whitelist', '10'], '').get_parsed_arg() + assert args.dynamic_whitelist == 10 + + +def test_parse_args_dynamic_whitelist_invalid_values() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['--dynamic-whitelist', 'abc'], '').get_parsed_arg() + + +def test_parse_timerange_incorrect() -> None: + assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200') + assert (('line', None), 200, None) == Arguments.parse_timerange('200-') + with pytest.raises(Exception, match=r'Incorrect syntax.*'): + Arguments.parse_timerange('-') + + +def test_parse_args_backtesting_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['backtesting --ticker-interval'], '').get_parsed_arg() + + with pytest.raises(SystemExit, match=r'2'): + Arguments(['backtesting --ticker-interval', 'abc'], '').get_parsed_arg() + + +def test_parse_args_backtesting_custom() -> None: + args = [ + '-c', 'test_conf.json', + 'backtesting', + '--live', + '--ticker-interval', '1', + '--refresh-pairs-cached'] + call_args = Arguments(args, '').get_parsed_arg() + assert call_args.config == 'test_conf.json' + assert call_args.live is True + assert call_args.loglevel == logging.INFO + assert call_args.subparser == 'backtesting' + assert call_args.func is not None + assert call_args.ticker_interval == 1 + assert call_args.refresh_pairs is True + + +def test_parse_args_hyperopt_custom() -> None: + args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20'] + call_args = Arguments(args, '').get_parsed_arg() + assert call_args.config == 'test_conf.json' + assert call_args.epochs == 20 + assert call_args.loglevel == logging.INFO + assert call_args.subparser == 'hyperopt' + assert call_args.func is not None From 766ec5ad0f8216d9fd05230baabc36ec41dc7d50 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 12 Feb 2018 00:37:19 -0800 Subject: [PATCH 11/56] Update unit tests to be compatible with this refactoring Updated: - test_acl_pair to be compatible with FreqtradeBot() class - test_default_strategy.py to be compatible with Analyze() class --- freqtrade/tests/conftest.py | 22 ++++- .../tests/strategy/test_default_strategy.py | 4 +- freqtrade/tests/test_acl_pair.py | 84 +++++++++++-------- 3 files changed, 70 insertions(+), 40 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d101e6e9a..722017a15 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -11,7 +11,7 @@ from telegram import Chat, Message, Update from freqtrade.analyze import Analyze from freqtrade.constants import Constants -from freqtrade.strategy.strategy import Strategy +from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -24,6 +24,26 @@ def log_has(line, logs): False) +# Functions for recurrent object patching +def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: + """ + This function patch _init_modules() to not call dependencies + :param mocker: a Mocker object to apply patches + :param config: Config to pass to the bot + :return: None + """ + mocker.patch('freqtrade.fiat_convert.Pymarketcap', {'price_usd': 12345.0}) + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) + mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) + + return FreqtradeBot(config) + + @pytest.fixture(scope="module") def default_conf(): """ Returns validated configuration suitable for most tests """ diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index f23c1fa48..02b5630fc 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -2,13 +2,13 @@ import json import pytest from pandas import DataFrame from freqtrade.strategy.default_strategy import DefaultStrategy, class_name -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.analyze import Analyze @pytest.fixture def result(): with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) + return Analyze.parse_ticker_dataframe(json.load(data_file)) def test_default_strategy_class_name(): diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index b70596091..b5f52774d 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,6 +1,6 @@ -# pragma pylint: disable=missing-docstring,C0103 +# pragma pylint: disable=missing-docstring,C0103,protected-access -from freqtrade.main import refresh_whitelist, gen_pair_whitelist +import freqtrade.tests.conftest as tt # test tools # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine @@ -8,21 +8,22 @@ from freqtrade.main import refresh_whitelist, gen_pair_whitelist def whitelist_conf(): - return { - 'stake_currency': 'BTC', - 'exchange': { - 'pair_whitelist': [ - 'BTC_ETH', - 'BTC_TKN', - 'BTC_TRST', - 'BTC_SWT', - 'BTC_BCC' - ], - 'pair_blacklist': [ - 'BTC_BLK' - ], - }, - } + config = tt.default_conf() + + config['stake_currency'] = 'BTC' + config['exchange']['pair_whitelist'] = [ + 'BTC_ETH', + 'BTC_TKN', + 'BTC_TRST', + 'BTC_SWT', + 'BTC_BCC' + ] + + config['exchange']['pair_blacklist'] = [ + 'BTC_BLK' + ] + + return config def get_market_summaries(): @@ -86,11 +87,13 @@ def get_health_empty(): def test_refresh_market_pair_not_in_whitelist(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist( - conf['exchange']['pair_whitelist'] + ['BTC_XXX']) + + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + + mocker.patch('freqtrade.freqtradebot.exchange.get_wallet_health', get_health) + refreshedwhitelist = freqtradebot._refresh_whitelist( + conf['exchange']['pair_whitelist'] + ['BTC_XXX'] + ) # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed @@ -99,10 +102,11 @@ def test_refresh_market_pair_not_in_whitelist(mocker): def test_refresh_whitelist(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist']) + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + + mocker.patch('freqtrade.freqtradebot.exchange.get_wallet_health', get_health) + refreshedwhitelist = freqtradebot._refresh_whitelist(conf['exchange']['pair_whitelist']) + # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed @@ -111,26 +115,32 @@ def test_refresh_whitelist(mocker): def test_refresh_whitelist_dynamic(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health) - mocker.patch.multiple('freqtrade.main.exchange', - get_market_summaries=get_market_summaries) + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + get_wallet_health=get_health, + get_market_summaries=get_market_summaries + ) + # argument: use the whitelist dynamically by exchange-volume whitelist = ['BTC_TKN', 'BTC_ETH'] - refreshedwhitelist = refresh_whitelist( - gen_pair_whitelist(conf['stake_currency'])) + + refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot._gen_pair_whitelist(conf['stake_currency']) + ) + assert whitelist == refreshedwhitelist def test_refresh_whitelist_dynamic_empty(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health_empty) + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + mocker.patch('freqtrade.freqtradebot.exchange.get_wallet_health', get_health_empty) + # argument: use the whitelist dynamically by exchange-volume whitelist = [] conf['exchange']['pair_whitelist'] = [] - refresh_whitelist(whitelist) + freqtradebot._refresh_whitelist(whitelist) pairslist = conf['exchange']['pair_whitelist'] + assert set(whitelist) == set(pairslist) From f4ec073099ff9056b71beba79a11619da49b7273 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 12 Feb 2018 19:45:59 -0800 Subject: [PATCH 12/56] Move RPC and Telegram to classes --- freqtrade/rpc/__init__.py | 415 ------ freqtrade/rpc/rpc.py | 373 ++++++ freqtrade/rpc/rpc_manager.py | 60 + freqtrade/rpc/telegram.py | 796 ++++++------ freqtrade/tests/rpc/test_rpc.py | 645 ++++++---- freqtrade/tests/rpc/test_rpc_manager.py | 139 ++ freqtrade/tests/rpc/test_rpc_telegram.py | 1486 +++++++++++++--------- 7 files changed, 2238 insertions(+), 1676 deletions(-) create mode 100644 freqtrade/rpc/rpc.py create mode 100644 freqtrade/rpc/rpc_manager.py create mode 100644 freqtrade/tests/rpc/test_rpc_manager.py diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 163e0a8aa..e69de29bb 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -1,415 +0,0 @@ -import logging -import re -import arrow -from decimal import Decimal -from datetime import datetime, timedelta -from pandas import DataFrame -import sqlalchemy as sql -# from sqlalchemy import and_, func, text - -from freqtrade.persistence import Trade -from freqtrade.misc import State, get_state, update_state -from freqtrade import exchange -from freqtrade.fiat_convert import CryptoToFiatConverter -from . import telegram - -logger = logging.getLogger(__name__) - -_FIAT_CONVERT = CryptoToFiatConverter() -REGISTERED_MODULES = [] - - -def init(config: dict) -> None: - """ - Initializes all enabled rpc modules - :param config: config to use - :return: None - """ - - if config['telegram'].get('enabled', False): - logger.info('Enabling rpc.telegram ...') - REGISTERED_MODULES.append('telegram') - telegram.init(config) - - -def cleanup() -> None: - """ - Stops all enabled rpc modules - :return: None - """ - if 'telegram' in REGISTERED_MODULES: - logger.debug('Cleaning up rpc.telegram ...') - telegram.cleanup() - - -def send_msg(msg: str) -> None: - """ - Send given markdown message to all registered rpc modules - :param msg: message - :return: None - """ - logger.info(msg) - if 'telegram' in REGISTERED_MODULES: - telegram.send_msg(msg) - - -def shorten_date(_date): - """ - Trim the date so it fits on small screens - """ - new_date = re.sub('seconds?', 'sec', _date) - new_date = re.sub('minutes?', 'min', new_date) - new_date = re.sub('hours?', 'h', new_date) - new_date = re.sub('days?', 'd', new_date) - new_date = re.sub('^an?', '1', new_date) - return new_date - - -# -# Below follows the RPC backend -# it is prefixed with rpc_ -# to raise awareness that it is -# a remotely exposed function - - -def rpc_trade_status(): - # Fetch open trade - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if get_state() != State.RUNNING: - return (True, '*Status:* `trader is not running`') - elif not trades: - return (True, '*Status:* `no active trade`') - else: - result = [] - for trade in trades: - order = None - if trade.open_order_id: - order = exchange.get_order(trade.open_order_id) - # calculate profit and send message to user - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - current_profit = trade.calc_profit_percent(current_rate) - fmt_close_profit = '{:.2f}%'.format( - round(trade.close_profit * 100, 2) - ) if trade.close_profit else None - message = """ -*Trade ID:* `{trade_id}` -*Current Pair:* [{pair}]({market_url}) -*Open Since:* `{date}` -*Amount:* `{amount}` -*Open Rate:* `{open_rate:.8f}` -*Close Rate:* `{close_rate}` -*Current Rate:* `{current_rate:.8f}` -*Close Profit:* `{close_profit}` -*Current Profit:* `{current_profit:.2f}%` -*Open Order:* `{open_order}` - """.format( - trade_id=trade.id, - pair=trade.pair, - market_url=exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit=fmt_close_profit, - current_profit=round(current_profit * 100, 2), - open_order='({} rem={:.8f})'.format( - order['type'], order['remaining'] - ) if order else None, - ) - result.append(message) - return (False, result) - - -def rpc_status_table(): - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if get_state() != State.RUNNING: - return (True, '*Status:* `trader is not running`') - elif not trades: - return (True, '*Status:* `no active order`') - else: - trades_list = [] - for trade in trades: - # calculate profit and send message to user - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - trades_list.append([ - trade.id, - trade.pair, - shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) - ]) - - columns = ['ID', 'Pair', 'Since', 'Profit'] - df_statuses = DataFrame.from_records(trades_list, columns=columns) - df_statuses = df_statuses.set_index(columns[0]) - # The style used throughout is to return a tuple - # consisting of (error_occured?, result) - # Another approach would be to just return the - # result, or raise error - return (False, df_statuses) - - -def rpc_daily_profit(timescale, stake_currency, fiat_display_currency): - today = datetime.utcnow().date() - profit_days = {} - - if not (isinstance(timescale, int) and timescale > 0): - return (True, '*Daily [n]:* `must be an integer greater than 0`') - - fiat = _FIAT_CONVERT - for day in range(0, timescale): - profitday = today - timedelta(days=day) - trades = Trade.query \ - .filter(Trade.is_open.is_(False)) \ - .filter(Trade.close_date >= profitday)\ - .filter(Trade.close_date < (profitday + timedelta(days=1)))\ - .order_by(Trade.close_date)\ - .all() - curdayprofit = sum(trade.calc_profit() for trade in trades) - profit_days[profitday] = { - 'amount': format(curdayprofit, '.8f'), - 'trades': len(trades) - } - - stats = [ - [ - key, - '{value:.8f} {symbol}'.format( - value=float(value['amount']), - symbol=stake_currency - ), - '{value:.3f} {symbol}'.format( - value=fiat.convert_amount( - value['amount'], - stake_currency, - fiat_display_currency - ), - symbol=fiat_display_currency - ), - '{value} trade{s}'.format(value=value['trades'], s='' if value['trades'] < 2 else 's'), - ] - for key, value in profit_days.items() - ] - return (False, stats) - - -def rpc_trade_statistics(stake_currency, fiat_display_currency) -> None: - """ - :return: cumulative profit statistics. - """ - trades = Trade.query.order_by(Trade.id).all() - - profit_all_coin = [] - profit_all_percent = [] - profit_closed_coin = [] - profit_closed_percent = [] - durations = [] - - for trade in trades: - current_rate = None - - if not trade.open_rate: - continue - if trade.close_date: - durations.append((trade.close_date - trade.open_date).total_seconds()) - - if not trade.is_open: - profit_percent = trade.calc_profit_percent() - profit_closed_coin.append(trade.calc_profit()) - profit_closed_percent.append(profit_percent) - else: - # Get current rate - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - profit_percent = trade.calc_profit_percent(rate=current_rate) - - profit_all_coin.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate))) - profit_all_percent.append(profit_percent) - - best_pair = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .first() - - if not best_pair: - return (True, '*Status:* `no closed trade`') - - bp_pair, bp_rate = best_pair - - # FIX: we want to keep fiatconverter in a state/environment, - # doing this will utilize its caching functionallity, instead we reinitialize it here - fiat = _FIAT_CONVERT - # Prepare data to display - profit_closed_coin = round(sum(profit_closed_coin), 8) - profit_closed_percent = round(sum(profit_closed_percent) * 100, 2) - profit_closed_fiat = fiat.convert_amount( - profit_closed_coin, - stake_currency, - fiat_display_currency - ) - profit_all_coin = round(sum(profit_all_coin), 8) - profit_all_percent = round(sum(profit_all_percent) * 100, 2) - profit_all_fiat = fiat.convert_amount( - profit_all_coin, - stake_currency, - fiat_display_currency - ) - num = float(len(durations) or 1) - return (False, - {'profit_closed_coin': profit_closed_coin, - 'profit_closed_percent': profit_closed_percent, - 'profit_closed_fiat': profit_closed_fiat, - 'profit_all_coin': profit_all_coin, - 'profit_all_percent': profit_all_percent, - 'profit_all_fiat': profit_all_fiat, - 'trade_count': len(trades), - 'first_trade_date': arrow.get(trades[0].open_date).humanize(), - 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), - 'avg_duration': str(timedelta(seconds=sum(durations) / - num)).split('.')[0], - 'best_pair': bp_pair, - 'best_rate': round(bp_rate * 100, 2) - }) - - -def rpc_balance(fiat_display_currency): - """ - :return: current account balance per crypto - """ - balances = [ - c for c in exchange.get_balances() - if c['Balance'] or c['Available'] or c['Pending'] - ] - if not balances: - return (True, '`All balances are zero.`') - - output = [] - total = 0.0 - for currency in balances: - coin = currency['Currency'] - if coin == 'BTC': - currency["Rate"] = 1.0 - else: - if coin == 'USDT': - currency["Rate"] = 1.0 / exchange.get_ticker('USDT_BTC', False)['bid'] - else: - currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid'] - currency['BTC'] = currency["Rate"] * currency["Balance"] - total = total + currency['BTC'] - output.append({'currency': currency['Currency'], - 'available': currency['Available'], - 'balance': currency['Balance'], - 'pending': currency['Pending'], - 'est_btc': currency['BTC'] - }) - fiat = _FIAT_CONVERT - symbol = fiat_display_currency - value = fiat.convert_amount(total, 'BTC', symbol) - return (False, (output, total, symbol, value)) - - -def rpc_start(): - """ - Handler for start. - """ - if get_state() == State.RUNNING: - return (True, '*Status:* `already running`') - else: - update_state(State.RUNNING) - - -def rpc_stop(): - """ - Handler for stop. - """ - if get_state() == State.RUNNING: - update_state(State.STOPPED) - return (False, '`Stopping trader ...`') - else: - return (True, '*Status:* `already stopped`') - - -# FIX: no test for this!!!! -def rpc_forcesell(trade_id) -> None: - """ - Handler for forcesell . - Sells the given trade at current price - :return: error or None - """ - def _exec_forcesell(trade: Trade) -> str: - # Check if there is there is an open order - if trade.open_order_id: - order = exchange.get_order(trade.open_order_id) - - # Cancel open LIMIT_BUY orders and close trade - if order and not order['closed'] and order['type'] == 'LIMIT_BUY': - exchange.cancel_order(trade.open_order_id) - trade.close(order.get('rate') or trade.open_rate) - # TODO: sell amount which has been bought already - return - - # Ignore trades with an attached LIMIT_SELL order - if order and not order['closed'] and order['type'] == 'LIMIT_SELL': - return - - # Get current rate and execute sell - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - from freqtrade.main import execute_sell - execute_sell(trade, current_rate) - # ---- EOF def _exec_forcesell ---- - - if get_state() != State.RUNNING: - return (True, '`trader is not running`') - - if trade_id == 'all': - # Execute sell for all open orders - for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): - _exec_forcesell(trade) - return (False, '') - - # Query for trade - trade = Trade.query.filter(sql.and_( - Trade.id == trade_id, - Trade.is_open.is_(True) - )).first() - if not trade: - logger.warning('forcesell: Invalid argument received') - return (True, 'Invalid argument.') - - _exec_forcesell(trade) - return (False, '') - - -def rpc_performance() -> None: - """ - Handler for performance. - Shows a performance statistic from finished trades - """ - if get_state() != State.RUNNING: - return (True, '`trader is not running`') - - pair_rates = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum'), - sql.func.count(Trade.pair).label('count')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .all() - trades = [] - for (pair, rate, count) in pair_rates: - trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) - - return (False, trades) - - -def rpc_count() -> None: - """ - Returns the number of trades running - :return: None - """ - if get_state() != State.RUNNING: - return (True, '`trader is not running`') - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - return (False, trades) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py new file mode 100644 index 000000000..30acf0bf4 --- /dev/null +++ b/freqtrade/rpc/rpc.py @@ -0,0 +1,373 @@ +""" +This module contains class to define a RPC communications +""" + +import arrow +from decimal import Decimal +from datetime import datetime, timedelta +from pandas import DataFrame +import sqlalchemy as sql +from freqtrade.logger import Logger +from freqtrade.persistence import Trade +from freqtrade.state import State +from freqtrade import exchange +from freqtrade.misc import shorten_date + + +class RPC(object): + """ + RPC class can be used to have extra feature, like bot data, and access to DB data + """ + + def __init__(self, freqtrade) -> None: + """ + Initializes all enabled rpc modules + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + self.freqtrade = freqtrade + self.logger = Logger( + name=__name__, + level=self.freqtrade.config.get('loglevel') + ).get_logger() + + def rpc_trade_status(self) -> (bool, Trade): + """ + Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is + a remotely exposed function + :return: + """ + # Fetch open trade + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + if self.freqtrade.get_state() != State.RUNNING: + return (True, '*Status:* `trader is not running`') + elif not trades: + return (True, '*Status:* `no active trade`') + else: + result = [] + for trade in trades: + order = None + if trade.open_order_id: + order = exchange.get_order(trade.open_order_id) + # calculate profit and send message to user + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + current_profit = trade.calc_profit_percent(current_rate) + fmt_close_profit = '{:.2f}%'.format( + round(trade.close_profit * 100, 2) + ) if trade.close_profit else None + message = "*Trade ID:* `{trade_id}`\n" \ + "*Current Pair:* [{pair}]({market_url})\n" \ + "*Open Since:* `{date}`\n" \ + "*Amount:* `{amount}`\n" \ + "*Open Rate:* `{open_rate:.8f}`\n" \ + "*Close Rate:* `{close_rate}`\n" \ + "*Current Rate:* `{current_rate:.8f}`\n" \ + "*Close Profit:* `{close_profit}`\n" \ + "*Current Profit:* `{current_profit:.2f}%`\n" \ + "*Open Order:* `{open_order}`"\ + .format( + trade_id=trade.id, + pair=trade.pair, + market_url=exchange.get_pair_detail_url(trade.pair), + date=arrow.get(trade.open_date).humanize(), + open_rate=trade.open_rate, + close_rate=trade.close_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + close_profit=fmt_close_profit, + current_profit=round(current_profit * 100, 2), + open_order='({} rem={:.8f})'.format( + order['type'], order['remaining'] + ) if order else None, + ) + result.append(message) + return (False, result) + + def rpc_status_table(self) -> (bool, DataFrame): + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + if self.freqtrade.get_state() != State.RUNNING: + return (True, '*Status:* `trader is not running`') + elif not trades: + return (True, '*Status:* `no active order`') + else: + trades_list = [] + for trade in trades: + # calculate profit and send message to user + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + trades_list.append([ + trade.id, + trade.pair, + shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), + '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) + ]) + + columns = ['ID', 'Pair', 'Since', 'Profit'] + df_statuses = DataFrame.from_records(trades_list, columns=columns) + df_statuses = df_statuses.set_index(columns[0]) + # The style used throughout is to return a tuple + # consisting of (error_occured?, result) + # Another approach would be to just return the + # result, or raise error + return (False, df_statuses) + + def rpc_daily_profit(self, timescale, stake_currency, fiat_display_currency): + today = datetime.utcnow().date() + profit_days = {} + + if not (isinstance(timescale, int) and timescale > 0): + return (True, '*Daily [n]:* `must be an integer greater than 0`') + + fiat = self.freqtrade.fiat_converter + for day in range(0, timescale): + profitday = today - timedelta(days=day) + trades = Trade.query \ + .filter(Trade.is_open.is_(False)) \ + .filter(Trade.close_date >= profitday)\ + .filter(Trade.close_date < (profitday + timedelta(days=1)))\ + .order_by(Trade.close_date)\ + .all() + curdayprofit = sum(trade.calc_profit() for trade in trades) + profit_days[profitday] = { + 'amount': format(curdayprofit, '.8f'), + 'trades': len(trades) + } + + stats = [ + [ + key, + '{value:.8f} {symbol}'.format( + value=float(value['amount']), + symbol=stake_currency + ), + '{value:.3f} {symbol}'.format( + value=fiat.convert_amount( + value['amount'], + stake_currency, + fiat_display_currency + ), + symbol=fiat_display_currency + ), + '{value} trade{s}'.format(value=value['trades'], s='' if value['trades'] < 2 else 's'), + ] + for key, value in profit_days.items() + ] + return (False, stats) + + def rpc_trade_statistics(self, stake_currency, fiat_display_currency) -> None: + """ + :return: cumulative profit statistics. + """ + trades = Trade.query.order_by(Trade.id).all() + + profit_all_coin = [] + profit_all_percent = [] + profit_closed_coin = [] + profit_closed_percent = [] + durations = [] + + for trade in trades: + current_rate = None + + if not trade.open_rate: + continue + if trade.close_date: + durations.append((trade.close_date - trade.open_date).total_seconds()) + + if not trade.is_open: + profit_percent = trade.calc_profit_percent() + profit_closed_coin.append(trade.calc_profit()) + profit_closed_percent.append(profit_percent) + else: + # Get current rate + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + profit_percent = trade.calc_profit_percent(rate=current_rate) + + profit_all_coin.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate))) + profit_all_percent.append(profit_percent) + + best_pair = Trade.session.query(Trade.pair, + sql.func.sum(Trade.close_profit).label('profit_sum')) \ + .filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(sql.text('profit_sum DESC')) \ + .first() + + if not best_pair: + return (True, '*Status:* `no closed trade`') + + bp_pair, bp_rate = best_pair + + # FIX: we want to keep fiatconverter in a state/environment, + # doing this will utilize its caching functionallity, instead we reinitialize it here + fiat = self.freqtrade.fiat_converter + # Prepare data to display + profit_closed_coin = round(sum(profit_closed_coin), 8) + profit_closed_percent = round(sum(profit_closed_percent) * 100, 2) + profit_closed_fiat = fiat.convert_amount( + profit_closed_coin, + stake_currency, + fiat_display_currency + ) + profit_all_coin = round(sum(profit_all_coin), 8) + profit_all_percent = round(sum(profit_all_percent) * 100, 2) + profit_all_fiat = fiat.convert_amount( + profit_all_coin, + stake_currency, + fiat_display_currency + ) + num = float(len(durations) or 1) + return ( + False, + { + 'profit_closed_coin': profit_closed_coin, + 'profit_closed_percent': profit_closed_percent, + 'profit_closed_fiat': profit_closed_fiat, + 'profit_all_coin': profit_all_coin, + 'profit_all_percent': profit_all_percent, + 'profit_all_fiat': profit_all_fiat, + 'trade_count': len(trades), + 'first_trade_date': arrow.get(trades[0].open_date).humanize(), + 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), + 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], + 'best_pair': bp_pair, + 'best_rate': round(bp_rate * 100, 2) + } + ) + + def rpc_balance(self, fiat_display_currency): + """ + :return: current account balance per crypto + """ + balances = [ + c for c in exchange.get_balances() + if c['Balance'] or c['Available'] or c['Pending'] + ] + if not balances: + return (True, '`All balances are zero.`') + + output = [] + total = 0.0 + for currency in balances: + coin = currency['Currency'] + if coin == 'BTC': + currency["Rate"] = 1.0 + else: + if coin == 'USDT': + currency["Rate"] = 1.0 / exchange.get_ticker('USDT_BTC', False)['bid'] + else: + currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid'] + currency['BTC'] = currency["Rate"] * currency["Balance"] + total = total + currency['BTC'] + output.append({'currency': currency['Currency'], + 'available': currency['Available'], + 'balance': currency['Balance'], + 'pending': currency['Pending'], + 'est_btc': currency['BTC'] + }) + fiat = self.freqtrade.fiat_converter + symbol = fiat_display_currency + value = fiat.convert_amount(total, 'BTC', symbol) + return (False, (output, total, symbol, value)) + + def rpc_start(self) -> (bool, str): + """ + Handler for start. + """ + if self.freqtrade.get_state() == State.RUNNING: + return (True, '*Status:* `already running`') + else: + self.freqtrade.update_state(State.RUNNING) + return (False, '`Starting trader ...`') + + def rpc_stop(self) -> (bool, str): + """ + Handler for stop. + """ + if self.freqtrade.get_state() == State.RUNNING: + self.freqtrade.update_state(State.STOPPED) + return (False, '`Stopping trader ...`') + else: + return (True, '*Status:* `already stopped`') + + # FIX: no test for this!!!! + def rpc_forcesell(self, trade_id) -> None: + """ + Handler for forcesell . + Sells the given trade at current price + :return: error or None + """ + def _exec_forcesell(trade: Trade) -> str: + # Check if there is there is an open order + if trade.open_order_id: + order = exchange.get_order(trade.open_order_id) + + # Cancel open LIMIT_BUY orders and close trade + if order and not order['closed'] and order['type'] == 'LIMIT_BUY': + exchange.cancel_order(trade.open_order_id) + trade.close(order.get('rate') or trade.open_rate) + # TODO: sell amount which has been bought already + return + + # Ignore trades with an attached LIMIT_SELL order + if order and not order['closed'] and order['type'] == 'LIMIT_SELL': + return + + # Get current rate and execute sell + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + self.freqtrade.execute_sell(trade, current_rate) + # ---- EOF def _exec_forcesell ---- + + if self.freqtrade.get_state() != State.RUNNING: + return (True, '`trader is not running`') + + if trade_id == 'all': + # Execute sell for all open orders + for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): + _exec_forcesell(trade) + return (False, '') + + # Query for trade + trade = Trade.query.filter( + sql.and_( + Trade.id == trade_id, + Trade.is_open.is_(True) + ) + ).first() + if not trade: + self.logger.warning('forcesell: Invalid argument received') + return (True, 'Invalid argument.') + + _exec_forcesell(trade) + return (False, '') + + def rpc_performance(self) -> None: + """ + Handler for performance. + Shows a performance statistic from finished trades + """ + if self.freqtrade.get_state() != State.RUNNING: + return (True, '`trader is not running`') + + pair_rates = Trade.session.query(Trade.pair, + sql.func.sum(Trade.close_profit).label('profit_sum'), + sql.func.count(Trade.pair).label('count')) \ + .filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(sql.text('profit_sum DESC')) \ + .all() + trades = [] + for (pair, rate, count) in pair_rates: + trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) + + return (False, trades) + + def rpc_count(self) -> None: + """ + Returns the number of trades running + :return: None + """ + if self.freqtrade.get_state() != State.RUNNING: + return (True, '`trader is not running`') + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + return (False, trades) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py new file mode 100644 index 000000000..5987d8c4d --- /dev/null +++ b/freqtrade/rpc/rpc_manager.py @@ -0,0 +1,60 @@ +""" +This module contains class to manage RPC communications (Telegram, Slack, ...) +""" + +from freqtrade.logger import Logger +from freqtrade.rpc.telegram import Telegram + + +class RPCManager(object): + """ + Class to manage RPC objects (Telegram, Slack, ...) + """ + + def __init__(self, freqtrade) -> None: + """ + Initializes all enabled rpc modules + :param config: config to use + :return: None + """ + self.freqtrade = freqtrade + + # Init the logger + self.logger = Logger( + name=__name__, + level=self.freqtrade.config.get('loglevel') + ).get_logger() + + self.registered_modules = [] + self.telegram = None + self._init() + + def _init(self): + """ + Init RPC modules + :return: + """ + if self.freqtrade.config['telegram'].get('enabled', False): + self.logger.info('Enabling rpc.telegram ...') + self.registered_modules.append('telegram') + self.telegram = Telegram(self.freqtrade) + + def cleanup(self) -> None: + """ + Stops all enabled rpc modules + :return: None + """ + if 'telegram' in self.registered_modules: + self.logger.info('Cleaning up rpc.telegram ...') + self.registered_modules.remove('telegram') + self.telegram.cleanup() + + def send_msg(self, msg: str) -> None: + """ + Send given markdown message to all registered rpc modules + :param msg: message + :return: None + """ + self.logger.info(msg) + if 'telegram' in self.registered_modules: + self.telegram.send_msg(msg) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ea170baa1..c0f5f815b 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,94 +1,14 @@ -import logging -from typing import Any, Callable +""" +This module manage Telegram communication +""" +from typing import Any, Callable +from freqtrade.rpc.rpc import RPC from tabulate import tabulate from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater - -from freqtrade.rpc.__init__ import (rpc_status_table, - rpc_trade_status, - rpc_daily_profit, - rpc_trade_statistics, - rpc_balance, - rpc_start, - rpc_stop, - rpc_forcesell, - rpc_performance, - rpc_count, - ) - -from freqtrade import __version__ - - -# Remove noisy log messages -logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) -logging.getLogger('telegram').setLevel(logging.INFO) -logger = logging.getLogger(__name__) - -_UPDATER: Updater = None -_CONF = {} - - -def init(config: dict) -> None: - """ - Initializes this module with the given config, - registers all known command handlers - and starts polling for message updates - :param config: config to use - :return: None - """ - global _UPDATER - - _CONF.update(config) - if not is_enabled(): - return - - _UPDATER = Updater(token=config['telegram']['token'], workers=0) - - # Register command handler and start telegram message polling - handles = [ - CommandHandler('status', _status), - CommandHandler('profit', _profit), - CommandHandler('balance', _balance), - CommandHandler('start', _start), - CommandHandler('stop', _stop), - CommandHandler('forcesell', _forcesell), - CommandHandler('performance', _performance), - CommandHandler('daily', _daily), - CommandHandler('count', _count), - CommandHandler('help', _help), - CommandHandler('version', _version), - ] - for handle in handles: - _UPDATER.dispatcher.add_handler(handle) - _UPDATER.start_polling( - clean=True, - bootstrap_retries=-1, - timeout=30, - read_latency=60, - ) - logger.info( - 'rpc.telegram is listening for following commands: %s', - [h.command for h in handles] - ) - - -def cleanup() -> None: - """ - Stops all running telegram threads. - :return: None - """ - if not is_enabled(): - return - _UPDATER.stop() - - -def is_enabled() -> bool: - """ - Returns True if the telegram module is activated, False otherwise - """ - return bool(_CONF['telegram'].get('enabled', False)) +from freqtrade.__init__ import __version__ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: @@ -97,340 +17,424 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ :param command_handler: Telegram CommandHandler :return: decorated function """ - def wrapper(*args, **kwargs): + + #def wrapper(self, bot: Bot, update: Update): + def wrapper(self, *args, **kwargs): + update = kwargs.get('update') or args[1] # Reject unauthorized messages - chat_id = int(_CONF['telegram']['chat_id']) + chat_id = int(self._config['telegram']['chat_id']) + if int(update.message.chat_id) != chat_id: - logger.info('Rejected unauthorized message from: %s', update.message.chat_id) + self.logger.info( + 'Rejected unauthorized message from: %s', + update.message.chat_id + ) return wrapper - logger.info('Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id) + self.logger.info( + 'Executing handler: %s for chat_id: %s', + command_handler.__name__, + chat_id + ) try: - return command_handler(*args, **kwargs) + return command_handler(self, *args, **kwargs) except BaseException: - logger.exception('Exception occurred within Telegram module') + self.logger.exception('Exception occurred within Telegram module') + return wrapper - -@authorized_only -def _status(bot: Bot, update: Update) -> None: +class Telegram(RPC): """ - Handler for /status. - Returns the current TradeThread status - :param bot: telegram bot - :param update: message update - :return: None + Telegram, this class send messages to Telegram """ + def __init__(self, freqtrade) -> None: + """ + Init the Telegram call, and init the super class RPC + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + super().__init__(freqtrade) - # Check if additional parameters are passed - params = update.message.text.replace('/status', '').split(' ') \ - if update.message.text else [] - if 'table' in params: - _status_table(bot, update) - return + self._updater = Updater = None + self._config = freqtrade.config + self._init() - # Fetch open trade - (error, trades) = rpc_trade_status() - if error: - send_msg(trades, bot=bot) - else: - for trademsg in trades: - send_msg(trademsg, bot=bot) + def _init(self) -> None: + """ + Initializes this module with the given config, + registers all known command handlers + and starts polling for message updates + :param config: config to use + :return: None + """ + if not self.is_enabled(): + return + self._updater = Updater(token=self._config['telegram']['token'], workers=0) -@authorized_only -def _status_table(bot: Bot, update: Update) -> None: - """ - Handler for /status table. - Returns the current TradeThread status in table format - :param bot: telegram bot - :param update: message update - :return: None - """ - # Fetch open trade - (err, df_statuses) = rpc_status_table() - if err: - send_msg(df_statuses, bot=bot) - else: - message = tabulate(df_statuses, headers='keys', tablefmt='simple') - message = "
{}
".format(message) + # Register command handler and start telegram message polling + handles = [ + CommandHandler('status', self._status), + CommandHandler('profit', self._profit), + CommandHandler('balance', self._balance), + CommandHandler('start', self._start), + CommandHandler('stop', self._stop), + CommandHandler('forcesell', self._forcesell), + CommandHandler('performance', self._performance), + CommandHandler('daily', self._daily), + CommandHandler('count', self._count), + CommandHandler('help', self._help), + CommandHandler('version', self._version), + ] + for handle in handles: + self._updater.dispatcher.add_handler(handle) + self._updater.start_polling( + clean=True, + bootstrap_retries=-1, + timeout=30, + read_latency=60, + ) + self.logger.info( + 'rpc.telegram is listening for following commands: %s', + [h.command for h in handles] + ) - send_msg(message, parse_mode=ParseMode.HTML) + def cleanup(self) -> None: + """ + Stops all running telegram threads. + :return: None + """ + if not self.is_enabled(): + return + import pprint + pprint.pprint(self._updater.stop.call_count) + self._updater.stop() -@authorized_only -def _daily(bot: Bot, update: Update) -> None: - """ - Handler for /daily - Returns a daily profit (in BTC) over the last n days. - :param bot: telegram bot - :param update: message update - :return: None - """ - try: - timescale = int(update.message.text.replace('/daily', '').strip()) - except (TypeError, ValueError): - timescale = 7 - (error, stats) = rpc_daily_profit(timescale, - _CONF['stake_currency'], - _CONF['fiat_display_currency']) - if error: - send_msg(stats, bot=bot) - else: - stats = tabulate(stats, - headers=[ - 'Day', - 'Profit {}'.format(_CONF['stake_currency']), - 'Profit {}'.format(_CONF['fiat_display_currency']) - ], - tablefmt='simple') - message = 'Daily Profit over the last {} days:\n
{}
'.format( - timescale, stats) - send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + def is_enabled(self) -> bool: + """ + Returns True if the telegram module is activated, False otherwise + """ + return bool(self._config.get('telegram', {}).get('enabled', False)) + @authorized_only + def _status(self, bot: Bot, update: Update) -> None: + """ + Handler for /status. + Returns the current TradeThread status + :param bot: telegram bot + :param update: message update + :return: None + """ -@authorized_only -def _profit(bot: Bot, update: Update) -> None: - """ - Handler for /profit. - Returns a cumulative profit statistics. - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, stats) = rpc_trade_statistics(_CONF['stake_currency'], - _CONF['fiat_display_currency']) - if error: - send_msg(stats, bot=bot) - return + # Check if additional parameters are passed + params = update.message.text.replace('/status', '').split(' ') \ + if update.message.text else [] + if 'table' in params: + self._status_table(bot, update) + return - # Message to display - markdown_msg = """ -*ROI:* Close trades - ∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)` - ∙ `{profit_closed_fiat:.3f} {fiat}` -*ROI:* All trades - ∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)` - ∙ `{profit_all_fiat:.3f} {fiat}` + # Fetch open trade + (error, trades) = self.rpc_trade_status() + if error: + self.send_msg(trades, bot=bot) + else: + for trademsg in trades: + self.send_msg(trademsg, bot=bot) -*Total Trade Count:* `{trade_count}` -*First Trade opened:* `{first_trade_date}` -*Latest Trade opened:* `{latest_trade_date}` -*Avg. Duration:* `{avg_duration}` -*Best Performing:* `{best_pair}: {best_rate:.2f}%` - """.format( - coin=_CONF['stake_currency'], - fiat=_CONF['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) - send_msg(markdown_msg, bot=bot) + @authorized_only + def _status_table(self, bot: Bot, update: Update) -> None: + """ + Handler for /status table. + Returns the current TradeThread status in table format + :param bot: telegram bot + :param update: message update + :return: None + """ + # Fetch open trade + (err, df_statuses) = self.rpc_status_table() + if err: + self.send_msg(df_statuses, bot=bot) + else: + message = tabulate(df_statuses, headers='keys', tablefmt='simple') + message = "
{}
".format(message) + self.send_msg(message, parse_mode=ParseMode.HTML) -@authorized_only -def _balance(bot: Bot, update: Update) -> None: - """ - Handler for /balance - """ - (error, result) = rpc_balance(_CONF['fiat_display_currency']) - if error: - send_msg('`All balances are zero.`') - return - - (currencys, total, symbol, value) = result - output = '' - for currency in currencys: - output += """*Currency*: {currency} -*Available*: {available} -*Balance*: {balance} -*Pending*: {pending} -*Est. BTC*: {est_btc: .8f} -""".format(**currency) - - output += """*Estimated Value*: -*BTC*: {0: .8f} -*{1}*: {2: .2f} -""".format(total, symbol, value) - send_msg(output) - - -@authorized_only -def _start(bot: Bot, update: Update) -> None: - """ - Handler for /start. - Starts TradeThread - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, msg) = rpc_start() - if error: - send_msg(msg, bot=bot) - - -@authorized_only -def _stop(bot: Bot, update: Update) -> None: - """ - Handler for /stop. - Stops TradeThread - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, msg) = rpc_stop() - send_msg(msg, bot=bot) - - -# FIX: no test for this!!!! -@authorized_only -def _forcesell(bot: Bot, update: Update) -> None: - """ - Handler for /forcesell . - Sells the given trade at current price - :param bot: telegram bot - :param update: message update - :return: None - """ - - trade_id = update.message.text.replace('/forcesell', '').strip() - (error, message) = rpc_forcesell(trade_id) - if error: - send_msg(message, bot=bot) - return - - -@authorized_only -def _performance(bot: Bot, update: Update) -> None: - """ - Handler for /performance. - Shows a performance statistic from finished trades - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, trades) = rpc_performance() - if error: - send_msg(trades, bot=bot) - return - - stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( - index=i + 1, - pair=trade['pair'], - profit=trade['profit'], - count=trade['count'] - ) for i, trade in enumerate(trades)) - message = 'Performance:\n{}'.format(stats) - send_msg(message, parse_mode=ParseMode.HTML) - - -@authorized_only -def _count(bot: Bot, update: Update) -> None: - """ - Handler for /count. - Returns the number of trades running - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, trades) = rpc_count() - if error: - send_msg(trades, bot=bot) - return - - message = tabulate({ - 'current': [len(trades)], - 'max': [_CONF['max_open_trades']] - }, headers=['current', 'max'], tablefmt='simple') - message = "
{}
".format(message) - logger.debug(message) - send_msg(message, parse_mode=ParseMode.HTML) - - -@authorized_only -def _help(bot: Bot, update: Update) -> None: - """ - Handler for /help. - Show commands of the bot - :param bot: telegram bot - :param update: message update - :return: None - """ - message = """ -*/start:* `Starts the trader` -*/stop:* `Stops the trader` -*/status [table]:* `Lists all open trades` - *table :* `will display trades in a table` -*/profit:* `Lists cumulative profit from all finished trades` -*/forcesell |all:* `Instantly sells the given trade or all trades, regardless of profit` -*/performance:* `Show performance of each finished trade grouped by pair` -*/daily :* `Shows profit or loss per day, over the last n days` -*/count:* `Show number of trades running compared to allowed number of trades` -*/balance:* `Show account balance per currency` -*/help:* `This help message` -*/version:* `Show version` - """ - send_msg(message, bot=bot) - - -@authorized_only -def _version(bot: Bot, update: Update) -> None: - """ - Handler for /version. - Show version information - :param bot: telegram bot - :param update: message update - :return: None - """ - send_msg('*Version:* `{}`'.format(__version__), bot=bot) - - -def send_msg(msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: - """ - Send given markdown message - :param msg: message - :param bot: alternative bot - :param parse_mode: telegram parse mode - :return: None - """ - if not is_enabled(): - return - - bot = bot or _UPDATER.bot - - keyboard = [['/daily', '/profit', '/balance'], - ['/status', '/status table', '/performance'], - ['/count', '/start', '/stop', '/help']] - - reply_markup = ReplyKeyboardMarkup(keyboard) - - try: + @authorized_only + def _daily(self, bot: Bot, update: Update) -> None: + """ + Handler for /daily + Returns a daily profit (in BTC) over the last n days. + :param bot: telegram bot + :param update: message update + :return: None + """ try: - bot.send_message( - _CONF['telegram']['chat_id'], msg, - parse_mode=parse_mode, reply_markup=reply_markup + timescale = int(update.message.text.replace('/daily', '').strip()) + except (TypeError, ValueError): + timescale = 7 + (error, stats) = self.rpc_daily_profit( + timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + if error: + self.send_msg(stats, bot=bot) + else: + stats = tabulate(stats, + headers=[ + 'Day', + 'Profit {}'.format(self._config['stake_currency']), + 'Profit {}'.format(self._config['fiat_display_currency']) + ], + tablefmt='simple') + message = 'Daily Profit over the last {} days:\n
{}
'\ + .format( + timescale, + stats + ) + self.send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + + @authorized_only + def _profit(self, bot: Bot, update: Update) -> None: + """ + Handler for /profit. + Returns a cumulative profit statistics. + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, stats) = self.rpc_trade_statistics( + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + if error: + self.send_msg(stats, bot=bot) + return + + # Message to display + markdown_msg = "*ROI:* Close trades\n" \ + "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ + "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ + "*ROI:* All trades\n" \ + "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ + "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ + "*Total Trade Count:* `{trade_count}`\n" \ + "*First Trade opened:* `{first_trade_date}`\n" \ + "*Latest Trade opened:* `{latest_trade_date}`\n" \ + "*Avg. Duration:* `{avg_duration}`\n" \ + "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ + .format( + coin=self._config['stake_currency'], + fiat=self._config['fiat_display_currency'], + profit_closed_coin=stats['profit_closed_coin'], + profit_closed_percent=stats['profit_closed_percent'], + profit_closed_fiat=stats['profit_closed_fiat'], + profit_all_coin=stats['profit_all_coin'], + profit_all_percent=stats['profit_all_percent'], + profit_all_fiat=stats['profit_all_fiat'], + trade_count=stats['trade_count'], + first_trade_date=stats['first_trade_date'], + latest_trade_date=stats['latest_trade_date'], + avg_duration=stats['avg_duration'], + best_pair=stats['best_pair'], + best_rate=stats['best_rate'] + ) + self.send_msg(markdown_msg, bot=bot) + + @authorized_only + def _balance(self, bot: Bot, update: Update) -> None: + """ + Handler for /balance + """ + (error, result) = self.rpc_balance(self._config['fiat_display_currency']) + if error: + self.send_msg('`All balances are zero.`') + return + + (currencys, total, symbol, value) = result + output = '' + for currency in currencys: + output += """*Currency*: {currency} + *Available*: {available} + *Balance*: {balance} + *Pending*: {pending} + *Est. BTC*: {est_btc: .8f} + """.format(**currency) + + output += """*Estimated Value*: + *BTC*: {0: .8f} + *{1}*: {2: .2f} + """.format(total, symbol, value) + self.send_msg(output) + + @authorized_only + def _start(self, bot: Bot, update: Update) -> None: + """ + Handler for /start. + Starts TradeThread + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, msg) = self.rpc_start() + if error: + self.send_msg(msg, bot=bot) + + @authorized_only + def _stop(self, bot: Bot, update: Update) -> None: + """ + Handler for /stop. + Stops TradeThread + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, msg) = self.rpc_stop() + self.send_msg(msg, bot=bot) + + # FIX: no test for this!!!! + @authorized_only + def _forcesell(self, bot: Bot, update: Update) -> None: + """ + Handler for /forcesell . + Sells the given trade at current price + :param bot: telegram bot + :param update: message update + :return: None + """ + + trade_id = update.message.text.replace('/forcesell', '').strip() + (error, message) = self.rpc_forcesell(trade_id) + if error: + self.send_msg(message, bot=bot) + return + + @authorized_only + def _performance(self, bot: Bot, update: Update) -> None: + """ + Handler for /performance. + Shows a performance statistic from finished trades + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, trades) = self.rpc_performance() + if error: + self.send_msg(trades, bot=bot) + return + + stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( + index=i + 1, + pair=trade['pair'], + profit=trade['profit'], + count=trade['count'] + ) for i, trade in enumerate(trades)) + message = 'Performance:\n{}'.format(stats) + self.send_msg(message, parse_mode=ParseMode.HTML) + + @authorized_only + def _count(self, bot: Bot, update: Update) -> None: + """ + Handler for /count. + Returns the number of trades running + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, trades) = self.rpc_count() + if error: + self.send_msg(trades, bot=bot) + return + + message = tabulate({ + 'current': [len(trades)], + 'max': [self._config['max_open_trades']] + }, headers=['current', 'max'], tablefmt='simple') + message = "
{}
".format(message) + self.logger.debug(message) + self.send_msg(message, parse_mode=ParseMode.HTML) + + @authorized_only + def _help(self, bot: Bot, update: Update) -> None: + """ + Handler for /help. + Show commands of the bot + :param bot: telegram bot + :param update: message update + :return: None + """ + message = "*/start:* `Starts the trader`\n" \ + "*/stop:* `Stops the trader`\n" \ + "*/status [table]:* `Lists all open trades`\n" \ + " *table :* `will display trades in a table`\n" \ + "*/profit:* `Lists cumulative profit from all finished trades`\n" \ + "*/forcesell |all:* `Instantly sells the given trade or all trades, regardless of profit`\n" \ + "*/performance:* `Show performance of each finished trade grouped by pair`\n" \ + "*/daily :* `Shows profit or loss per day, over the last n days`\n" \ + "*/count:* `Show number of trades running compared to allowed number of trades`\n" \ + "*/balance:* `Show account balance per currency`\n" \ + "*/help:* `This help message`\n" \ + "*/version:* `Show version`" + + self.send_msg(message, bot=bot) + + @authorized_only + def _version(self, bot: Bot, update: Update) -> None: + """ + Handler for /version. + Show version information + :param bot: telegram bot + :param update: message update + :return: None + """ + self.send_msg('*Version:* `{}`'.format(__version__), bot=bot) + + def send_msg(self, msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + """ + Send given markdown message + :param msg: message + :param bot: alternative bot + :param parse_mode: telegram parse mode + :return: None + """ + if not self.is_enabled(): + return + + bot = bot or self._updater.bot + + keyboard = [['/daily', '/profit', '/balance'], + ['/status', '/status table', '/performance'], + ['/count', '/start', '/stop', '/help']] + + reply_markup = ReplyKeyboardMarkup(keyboard) + + try: + try: + bot.send_message( + self._config['telegram']['chat_id'], + text=msg, + parse_mode=parse_mode, + reply_markup=reply_markup + ) + except NetworkError as network_err: + # Sometimes the telegram server resets the current connection, + # if this is the case we send the message again. + self.logger.warning( + 'Got Telegram NetworkError: %s! Trying one more time.', + network_err.message + ) + bot.send_message( + self._config['telegram']['chat_id'], + text=msg, + parse_mode=parse_mode, + reply_markup=reply_markup + ) + except TelegramError as telegram_err: + self.logger.warning( + 'Got TelegramError: %s! Giving up on that message.', + telegram_err.message ) - except NetworkError as network_err: - # Sometimes the telegram server resets the current connection, - # if this is the case we send the message again. - logger.warning( - 'Got Telegram NetworkError: %s! Trying one more time.', - network_err.message - ) - bot.send_message( - _CONF['telegram']['chat_id'], msg, - parse_mode=parse_mode, reply_markup=reply_markup - ) - except TelegramError as telegram_err: - logger.warning('Got TelegramError: %s! Giving up on that message.', telegram_err.message) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 06994ce8e..e9d77b1f5 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -1,204 +1,133 @@ -# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 +# pragma pylint: disable=invalid-sequence-index, invalid-name, too-many-arguments + +""" +Unit test file for rpc/rpc.py +""" + from datetime import datetime -from copy import deepcopy from unittest.mock import MagicMock + from sqlalchemy import create_engine -from freqtrade.rpc import init, cleanup, send_msg +from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -import freqtrade.main as main -import freqtrade.misc as misc -import freqtrade.rpc as rpc +from freqtrade.rpc.rpc import RPC +from freqtrade.state import State +from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_pymarketcap -def prec_satoshi(a, b): +# Functions for recurrent object patching +def prec_satoshi(a, b) -> float: """ :return: True if A and B differs less than one satoshi. """ return abs(a - b) < 0.00000001 -def test_init_telegram_enabled(default_conf, mocker): - module_list = [] - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', module_list) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.init', MagicMock()) +# Unit tests +def test_rpc_trade_status(default_conf, ticker, mocker) -> None: + """ + Test rpc_trade_status() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) - init(default_conf) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) - assert telegram_mock.call_count == 1 - assert 'telegram' in module_list - - -def test_init_telegram_disabled(default_conf, mocker): - module_list = [] - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', module_list) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.init', MagicMock()) - - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - init(conf) - - assert telegram_mock.call_count == 0 - assert 'telegram' not in module_list - - -def test_cleanup_telegram_enabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', ['telegram']) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.cleanup', MagicMock()) - cleanup() - assert telegram_mock.call_count == 1 - - -def test_cleanup_telegram_disabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', []) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.cleanup', MagicMock()) - cleanup() - assert telegram_mock.call_count == 0 - - -def test_send_msg_telegram_enabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', ['telegram']) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.send_msg', MagicMock()) - send_msg('test') - assert telegram_mock.call_count == 1 - - -def test_send_msg_telegram_disabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', []) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.send_msg', MagicMock()) - send_msg('test') - assert telegram_mock.call_count == 0 - - -def test_rpc_forcesell(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - cancel_order_mock = MagicMock() - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - cancel_order=cancel_order_mock, - get_order=MagicMock(return_value={ - 'closed': True, - 'type': 'LIMIT_BUY', - })) - main.init(default_conf, create_engine('sqlite://')) - - misc.update_state(misc.State.STOPPED) - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' - misc.update_state(misc.State.RUNNING) - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == 'Invalid argument.' - - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' - - main.create_trade(0.001, 5) - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' - - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' - - misc.update_state(misc.State.STOPPED) - - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' - - (error, res) = rpc.rpc_forcesell('all') - assert error - assert res == '`trader is not running`' - - misc.update_state(misc.State.RUNNING) - - assert cancel_order_mock.call_count == 0 - # make an limit-buy open trade - mocker.patch.multiple('freqtrade.exchange', - get_order=MagicMock(return_value={ - 'closed': None, - 'type': 'LIMIT_BUY' - })) - # check that the trade is called, which is done - # by ensuring exchange.cancel_order is called - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' - assert cancel_order_mock.call_count == 1 - - main.create_trade(0.001, 5) - # make an limit-sell open trade - mocker.patch.multiple('freqtrade.exchange', - get_order=MagicMock(return_value={ - 'closed': None, - 'type': 'LIMIT_SELL' - })) - (error, res) = rpc.rpc_forcesell('2') - assert not error - assert res == '' - # status quo, no exchange calls - assert cancel_order_mock.call_count == 1 - - -def test_rpc_trade_status(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - main.init(default_conf, create_engine('sqlite://')) - - misc.update_state(misc.State.STOPPED) + freqtradebot.update_state(State.STOPPED) (error, result) = rpc.rpc_trade_status() assert error - assert result.find('trader is not running') >= 0 + assert 'trader is not running' in result - misc.update_state(misc.State.RUNNING) + freqtradebot.update_state(State.RUNNING) (error, result) = rpc.rpc_trade_status() assert error - assert result.find('no active trade') >= 0 + assert 'no active trade' in result - main.create_trade(0.001, 5) + freqtradebot.create_trade(0.001, 5) (error, result) = rpc.rpc_trade_status() assert not error trade = result[0] + + result_message = [ + '*Trade ID:* `1`\n' + '*Current Pair:* ' + '[BTC_ETH](https://www.bittrex.com/Market/Index?MarketName=BTC-ETH)\n' + '*Open Since:* `just now`\n' + '*Amount:* `90.99181074`\n' + '*Open Rate:* `0.00001099`\n' + '*Close Rate:* `None`\n' + '*Current Rate:* `0.00001098`\n' + '*Close Profit:* `None`\n' + '*Current Profit:* `-0.59%`\n' + '*Open Order:* `(LIMIT_BUY rem=0.00000000)`' + ] + assert result == result_message assert trade.find('[BTC_ETH]') >= 0 -def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - main.init(default_conf, create_engine('sqlite://')) +def test_rpc_status_table(default_conf, ticker, mocker) -> None: + """ + Test rpc_status_table() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + (error, result) = rpc.rpc_status_table() + assert error + assert '*Status:* `trader is not running`' in result + + freqtradebot.update_state(State.RUNNING) + (error, result) = rpc.rpc_status_table() + assert error + assert '*Status:* `no active order`' in result + + freqtradebot.create_trade(0.001, 5) + (error, result) = rpc.rpc_status_table() + assert 'just now' in result['Since'].all() + assert 'BTC_ETH' in result['Pair'].all() + assert '-0.59%' in result['Profit'].all() + + +def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker)\ + -> None: + """ + Test rpc_daily_profit() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] + rpc = RPC(freqtradebot) + # Create some test data - main.create_trade(0.001, 5) + freqtradebot.create_trade(0.001, 5) trade = Trade.query.first() assert trade @@ -210,8 +139,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_s # Try valid data update.message.text = '/daily 2' - (error, days) = rpc.rpc_daily_profit(7, stake_currency, - fiat_display_currency) + (error, days) = rpc.rpc_daily_profit(7, stake_currency, fiat_display_currency) assert not error assert len(days) == 7 for day in days: @@ -225,51 +153,57 @@ def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_s assert str(days[0][0]) == str(datetime.utcnow().date()) # Try invalid data - (error, days) = rpc.rpc_daily_profit(0, stake_currency, - fiat_display_currency) + (error, days) = rpc.rpc_daily_profit(0, stake_currency, fiat_display_currency) assert error assert days.find('must be an integer greater than 0') >= 0 def test_rpc_trade_statistics( - default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) + default_conf, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test rpc_trade_statistics() method + """ + patch_get_signal(mocker, (True, False)) + mocker.patch.multiple( + 'freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1}) + ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - main.init(default_conf, create_engine('sqlite://')) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] - (error, stats) = rpc.rpc_trade_statistics(stake_currency, - fiat_display_currency) + rpc = RPC(freqtradebot) + + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert error assert stats.find('no closed trade') >= 0 # Create some test data - main.create_trade(0.001, 5) + freqtradebot.create_trade(0.001, 5) trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) + # Update the ticker with a market going up - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() trade.is_open = False - (error, stats) = rpc.rpc_trade_statistics(stake_currency, - fiat_display_currency) + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert not error assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) @@ -287,34 +221,42 @@ def test_rpc_trade_statistics( # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) -def test_rpc_trade_statistics_closed( - default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) +def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, ticker_sell_up, limit_buy_order, + limit_sell_order): + """ + Test rpc_trade_statistics() method + """ + patch_get_signal(mocker, (True, False)) + mocker.patch.multiple( + 'freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1}) + ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - main.init(default_conf, create_engine('sqlite://')) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] + rpc = RPC(freqtradebot) + # Create some test data - main.create_trade(0.001, 5) + freqtradebot.create_trade(0.001, 5) trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) # Update the ticker with a market going up - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() trade.is_open = False @@ -322,8 +264,7 @@ def test_rpc_trade_statistics_closed( for trade in Trade.query.order_by(Trade.id).all(): trade.open_rate = None - (error, stats) = rpc.rpc_trade_statistics(stake_currency, - fiat_display_currency) + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert not error assert prec_satoshi(stats['profit_closed_coin'], 0) assert prec_satoshi(stats['profit_closed_percent'], 0) @@ -339,58 +280,224 @@ def test_rpc_trade_statistics_closed( assert prec_satoshi(stats['best_rate'], 6.2) -def test_rpc_balance_handle(default_conf, update, mocker): - mock_balance = [{ - 'Currency': 'BTC', - 'Balance': 10.0, - 'Available': 12.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'ETH', - 'Balance': 0.0, - 'Available': 0.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }] - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_balances=MagicMock(return_value=mock_balance)) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) +def test_rpc_balance_handle(default_conf, mocker): + """ + Test rpc_balance() method + """ + mock_balance = [ + { + 'Currency': 'BTC', + 'Balance': 10.0, + 'Available': 12.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'ETH', + 'Balance': 0.0, + 'Available': 0.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + } + ] + + patch_get_signal(mocker, (True, False)) + mocker.patch.multiple( + 'freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1}) + ) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_balances=MagicMock(return_value=mock_balance) + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency']) assert not error (trade, x, y, z) = res assert prec_satoshi(x, 10) assert prec_satoshi(z, 150000) - assert y == 'USD' + assert 'USD' in y assert len(trade) == 1 - assert trade[0]['currency'] == 'BTC' + assert 'BTC' in trade[0]['currency'] assert prec_satoshi(trade[0]['available'], 12) assert prec_satoshi(trade[0]['balance'], 10) assert prec_satoshi(trade[0]['pending'], 0) assert prec_satoshi(trade[0]['est_btc'], 10) -def test_performance_handle( - default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - main.init(default_conf, create_engine('sqlite://')) +def test_rpc_start(mocker, default_conf) -> None: + """ + Test rpc_start() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock() + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + freqtradebot.update_state(State.STOPPED) + + (error, result) = rpc.rpc_start() + assert not error + assert '`Starting trader ...`' in result + assert freqtradebot.get_state() == State.RUNNING + + (error, result) = rpc.rpc_start() + assert error + assert '*Status:* `already running`' in result + assert freqtradebot.get_state() == State.RUNNING + + +def test_rpc_stop(mocker, default_conf) -> None: + """ + Test rpc_stop() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock() + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + freqtradebot.update_state(State.RUNNING) + + (error, result) = rpc.rpc_stop() + assert not error + assert '`Stopping trader ...`' in result + assert freqtradebot.get_state() == State.STOPPED + + (error, result) = rpc.rpc_stop() + assert error + assert '*Status:* `already stopped`' in result + assert freqtradebot.get_state() == State.STOPPED + + +def test_rpc_forcesell(default_conf, ticker, mocker) -> None: + """ + Test rpc_forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + cancel_order=cancel_order_mock, + get_order=MagicMock( + return_value={ + 'closed': True, + 'type': 'LIMIT_BUY', + } + ) + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + (error, res) = rpc.rpc_forcesell(None) + assert error + assert res == '`trader is not running`' + + freqtradebot.update_state(State.RUNNING) + (error, res) = rpc.rpc_forcesell(None) + assert error + assert res == 'Invalid argument.' + + (error, res) = rpc.rpc_forcesell('all') + assert not error + assert res == '' + + freqtradebot.create_trade(0.001, 5) + (error, res) = rpc.rpc_forcesell('all') + assert not error + assert res == '' + + (error, res) = rpc.rpc_forcesell('1') + assert not error + assert res == '' + + freqtradebot.update_state(State.STOPPED) + (error, res) = rpc.rpc_forcesell(None) + assert error + assert res == '`trader is not running`' + + (error, res) = rpc.rpc_forcesell('all') + assert error + assert res == '`trader is not running`' + + freqtradebot.update_state(State.RUNNING) + assert cancel_order_mock.call_count == 0 + # make an limit-buy open trade + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_order', + return_value={ + 'closed': None, + 'type': 'LIMIT_BUY' + } + ) + # check that the trade is called, which is done + # by ensuring exchange.cancel_order is called + (error, res) = rpc.rpc_forcesell('1') + assert not error + assert res == '' + assert cancel_order_mock.call_count == 1 + + freqtradebot.create_trade(0.001, 5) + # make an limit-sell open trade + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_order', + return_value={ + 'closed': None, + 'type': 'LIMIT_SELL' + } + ) + (error, res) = rpc.rpc_forcesell('2') + assert not error + assert res == '' + # status quo, no exchange calls + assert cancel_order_mock.call_count == 1 + + +def test_performance_handle(default_conf, ticker, limit_buy_order, + limit_sell_order, mocker) -> None: + """ + Test rpc_performance() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_balances=MagicMock(return_value=ticker), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) # Create some test data - main.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) trade = Trade.query.first() assert trade @@ -408,3 +515,33 @@ def test_performance_handle( assert res[0]['pair'] == 'BTC_ETH' assert res[0]['count'] == 1 assert prec_satoshi(res[0]['profit'], 6.2) + + +def test_rpc_count(mocker, default_conf, ticker) -> None: + """ + Test rpc_count() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_balances=MagicMock(return_value=ticker), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + + (error, trades) = rpc.rpc_count() + nb_trades = len(trades) + assert not error + assert nb_trades == 0 + + # Create some test data + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + (error, trades) = rpc.rpc_count() + nb_trades = len(trades) + assert not error + assert nb_trades == 1 diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py new file mode 100644 index 000000000..e4a7d4cda --- /dev/null +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -0,0 +1,139 @@ +""" +Unit test file for rpc/rpc_manager.py +""" + +import logging +from copy import deepcopy +from unittest.mock import MagicMock + +from freqtrade.rpc.rpc_manager import RPCManager +from freqtrade.rpc.telegram import Telegram +import freqtrade.tests.conftest as tt # test tools + + +def test_rpc_manager_object() -> None: + """ + Test the Arguments object has the mandatory methods + :return: None + """ + assert hasattr(RPCManager, '_init') + assert hasattr(RPCManager, 'send_msg') + assert hasattr(RPCManager, 'cleanup') + + +def test__init__(mocker, default_conf) -> None: + """ + Test __init__() method + """ + init_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager._init', MagicMock()) + freqtradebot = tt.get_patched_freqtradebot(mocker, default_conf) + + rpc_manager = RPCManager(freqtradebot) + assert rpc_manager.freqtrade == freqtradebot + assert rpc_manager.registered_modules == [] + assert rpc_manager.telegram is None + assert init_mock.call_count == 1 + + +def test_init_telegram_disabled(mocker, default_conf, caplog) -> None: + """ + Test _init() method with Telegram disabled + """ + caplog.set_level(logging.DEBUG) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + rpc_manager = RPCManager(freqtradebot) + + assert not tt.log_has('Enabling rpc.telegram ...', caplog.record_tuples) + assert rpc_manager.registered_modules == [] + assert rpc_manager.telegram is None + + +def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: + """ + Test _init() method with Telegram enabled + """ + caplog.set_level(logging.DEBUG) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + + freqtradebot = tt.get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + + assert tt.log_has('Enabling rpc.telegram ...', caplog.record_tuples) + len_modules = len(rpc_manager.registered_modules) + assert len_modules == 1 + assert 'telegram' in rpc_manager.registered_modules + assert isinstance(rpc_manager.telegram, Telegram) + + +def test_cleanup_telegram_disabled(mocker, default_conf, caplog) -> None: + """ + Test cleanup() method with Telegram disabled + """ + caplog.set_level(logging.DEBUG) + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.cleanup', MagicMock()) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.cleanup() + + assert not tt.log_has('Cleaning up rpc.telegram ...', caplog.record_tuples) + assert telegram_mock.call_count == 0 + + +def test_cleanup_telegram_enabled(mocker, default_conf, caplog) -> None: + """ + Test cleanup() method with Telegram enabled + """ + caplog.set_level(logging.DEBUG) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.cleanup', MagicMock()) + + freqtradebot = tt.get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + + # Check we have Telegram as a registered modules + assert 'telegram' in rpc_manager.registered_modules + + rpc_manager.cleanup() + assert tt.log_has('Cleaning up rpc.telegram ...', caplog.record_tuples) + assert 'telegram' not in rpc_manager.registered_modules + assert telegram_mock.call_count == 1 + + +def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: + """ + Test send_msg() method with Telegram disabled + """ + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.send_msg('test') + + assert tt.log_has('test', caplog.record_tuples) + assert telegram_mock.call_count == 0 + + +def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: + """ + Test send_msg() method with Telegram disabled + """ + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + + freqtradebot = tt.get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.send_msg('test') + + assert tt.log_has('test', caplog.record_tuples) + assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 9a1dbcc69..5aaf13742 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1,148 +1,355 @@ -# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 -# pragma pylint: disable=unused-argument +# pragma pylint: disable=protected-access, unused-argument, invalid-name +# pragma pylint: disable=too-many-lines, too-many-arguments + +""" +Unit test file for rpc/telegram.py +""" + import re from datetime import datetime from random import randint from unittest.mock import MagicMock +from copy import deepcopy from sqlalchemy import create_engine from telegram import Update, Message, Chat from telegram.error import NetworkError from freqtrade import __version__ -from freqtrade.main import init, create_trade -from freqtrade.misc import update_state, State, get_state +from freqtrade.rpc.telegram import authorized_only +from freqtrade.freqtradebot import FreqtradeBot +from freqtrade.rpc.telegram import Telegram from freqtrade.persistence import Trade -from freqtrade.rpc import telegram -from freqtrade.rpc.telegram import authorized_only, is_enabled, send_msg, _status, _status_table, \ - _profit, _forcesell, _performance, _daily, _count, _start, _stop, _balance, _version, _help - -import freqtrade.rpc.telegram as tg +from freqtrade.state import State +from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_pymarketcap +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has +import freqtrade.tests.conftest as tt # test tools -def test_is_enabled(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) - default_conf['telegram']['enabled'] = False - assert is_enabled() is False +class DummyCls(Telegram): + """ + Dummy class for testing the Telegram @authorized_only decorator + """ + def __init__(self, freqtrade) -> None: + super().__init__(freqtrade) + self.state = {'called': False} + + @authorized_only + def dummy_handler(self, *args, **kwargs) -> None: + """ + Fake method that only change the state of the object + """ + self.state['called'] = True + + @authorized_only + def dummy_exception(self, *args, **kwargs) -> None: + """ + Fake method that throw an exception + """ + raise Exception('test') -def test_init_disabled(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) - default_conf['telegram']['enabled'] = False - telegram.init(default_conf) +def test__init__(default_conf, mocker) -> None: + """ + Test __init__() method + """ + mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + + telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) + assert telegram._updater is None + assert telegram._config == default_conf -def test_authorized_only(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) +def test_init(default_conf, mocker, caplog) -> None: + """ + Test _init() method + """ + start_polling = MagicMock() + mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock(return_value=start_polling)) + Telegram(get_patched_freqtradebot(mocker, default_conf)) + assert start_polling.call_count == 0 + + # number of handles registered + assert start_polling.dispatcher.add_handler.call_count == 11 + assert start_polling.start_polling.call_count == 1 + + message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ + "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \ + "['count'], ['help'], ['version']]" + + assert log_has(message_str, caplog.record_tuples) + + +def test_init_disabled(default_conf, mocker, caplog) -> None: + """ + Test _init() method when Telegram is disabled + """ + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + Telegram(get_patched_freqtradebot(mocker, conf)) + + message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ + "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \ + "['count'], ['help'], ['version']]" + + assert not log_has(message_str, caplog.record_tuples) + + +def test_cleanup(default_conf, mocker) -> None: + """ + Test cleanup() method + """ + updater_mock = MagicMock() + updater_mock.stop = MagicMock() + mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock) + + # not enabled + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + telegram.cleanup() + assert telegram._updater is None + assert updater_mock.call_count == 0 + assert not hasattr(telegram._updater, 'stop') + assert updater_mock.stop.call_count == 0 + + # enabled + conf['telegram']['enabled'] = True + telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + telegram.cleanup() + assert telegram._updater.stop.call_count == 1 + + +def test_is_enabled(default_conf, mocker) -> None: + """ + Test is_enabled() method + """ + mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) + + telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) + assert telegram.is_enabled() + + +def test_is_not_enabled(default_conf, mocker) -> None: + """ + Test is_enabled() method + """ + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + + assert not telegram.is_enabled() + + +def test_authorized_only(default_conf, caplog) -> None: + """ + Test authorized_only() method when we are authorized + """ chat = Chat(0, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) - state = {'called': False} - @authorized_only - def dummy_handler(*args, **kwargs) -> None: - state['called'] = True - - dummy_handler(MagicMock(), update) - assert state['called'] is True + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy.dummy_handler(bot=MagicMock(), update=update) + assert dummy.state['called'] is True + assert tt.log_has( + 'Executing handler: dummy_handler for chat_id: 0', + caplog.record_tuples + ) + assert not tt.log_has( + 'Rejected unauthorized message from: 0', + caplog.record_tuples + ) + assert not tt.log_has( + 'Exception occurred within Telegram module', + caplog.record_tuples + ) -def test_authorized_only_unauthorized(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) - +def test_authorized_only_unauthorized(default_conf, caplog) -> None: + """ + Test authorized_only() method when we are unauthorized + """ chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) - state = {'called': False} - @authorized_only - def dummy_handler(*args, **kwargs) -> None: - state['called'] = True - - dummy_handler(MagicMock(), update) - assert state['called'] is False + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy.dummy_handler(bot=MagicMock(), update=update) + assert dummy.state['called'] is False + assert not tt.log_has( + 'Executing handler: dummy_handler for chat_id: 3735928559', + caplog.record_tuples + ) + assert tt.log_has( + 'Rejected unauthorized message from: 3735928559', + caplog.record_tuples + ) + assert not tt.log_has( + 'Exception occurred within Telegram module', + caplog.record_tuples + ) -def test_authorized_only_exception(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) - +def test_authorized_only_exception(default_conf, caplog) -> None: + """ + Test authorized_only() method when an exception is thrown + """ update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), Chat(0, 0)) - @authorized_only - def dummy_handler(*args, **kwargs) -> None: - raise Exception('test') - - dummy_handler(MagicMock(), update) + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy.dummy_exception(bot=MagicMock(), update=update) + assert dummy.state['called'] is False + assert not tt.log_has( + 'Executing handler: dummy_handler for chat_id: 0', + caplog.record_tuples + ) + assert not tt.log_has( + 'Rejected unauthorized message from: 0', + caplog.record_tuples + ) + assert tt.log_has( + 'Exception occurred within Telegram module', + caplog.record_tuples + ) -def test_status_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_status(default_conf, update, mocker, ticker) -> None: + """ + Test _status() method + """ + update.message.chat.id = 123 + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + conf['telegram']['chat_id'] = 123 + + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) + status_table = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), + _status_table=status_table, + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - update_state(State.STOPPED) - _status(bot=MagicMock(), update=update) + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + for _ in range(3): + freqtradebot.create_trade(0.001, 5) + + telegram._status(bot=MagicMock(), update=update) + assert msg_mock.call_count == 3 + + update.message.text = MagicMock() + update.message.text.replace = MagicMock(return_value='table 2 3') + telegram._status(bot=MagicMock(), update=update) + assert status_table.call_count == 1 + + +def test_status_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _status() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + msg_mock = MagicMock() + status_table = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _status_table=status_table, + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'trader is not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - update_state(State.RUNNING) - _status(bot=MagicMock(), update=update) + freqtradebot.update_state(State.RUNNING) + telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) # Trigger status while we have a fulfilled order for the open trade - _status(bot=MagicMock(), update=update) + telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert '[BTC_ETH]' in msg_mock.call_args_list[0][0][0] -def test_status_table_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) +def test_status_table_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _status_table() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) mocker.patch.multiple( - 'freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_order_id')) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - _status_table(bot=MagicMock(), update=update) + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_order_id') + ) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + telegram._status_table(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'trader is not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - update_state(State.RUNNING) - _status_table(bot=MagicMock(), update=update) + freqtradebot.update_state(State.RUNNING) + telegram._status_table(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'no active order' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() # Create some test data - create_trade(15.0, int(default_conf['ticker_interval'])) + freqtradebot.create_trade(15.0, int(default_conf['ticker_interval'])) - _status_table(bot=MagicMock(), update=update) + telegram._status_table(bot=MagicMock(), update=update) text = re.sub('', '', msg_mock.call_args_list[-1][0][0]) line = text.split("\n") @@ -153,256 +360,35 @@ def test_status_table_handle(default_conf, update, ticker, mocker): assert msg_mock.call_count == 1 -def test_profit_handle( - default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_daily_handle(default_conf, update, ticker, limit_buy_order, + limit_sell_order, mocker) -> None: + """ + Test _daily() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch( + 'freqtrade.fiat_convert.CryptoToFiatConverter._find_price', + return_value=15000.0 + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - _profit(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'no closed trade' in msg_mock.call_args_list[0][0][0] - msg_mock.reset_mock() + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - trade = Trade.query.first() - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - - _profit(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'no closed trade' in msg_mock.call_args_list[-1][0][0] - msg_mock.reset_mock() - - # Update the ticker with a market going up - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - trade.update(limit_sell_order) - - trade.close_date = datetime.utcnow() - trade.is_open = False - - _profit(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert '*ROI:* Close trades' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] - assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] - - assert '*Best Performing:* `BTC_ETH: 6.20%`' in msg_mock.call_args_list[-1][0][0] - - -def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Increase the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - - update.message.text = '/forcesell 1' - _forcesell(bot=MagicMock(), update=update) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] - assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - # Decrease the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_down) - - trade = Trade.query.first() - assert trade - - update.message.text = '/forcesell 1' - _forcesell(bot=MagicMock(), update=update) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_forcesell_all_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - for _ in range(4): - create_trade(0.001, int(default_conf['ticker_interval'])) - rpc_mock.reset_mock() - - update.message.text = '/forcesell all' - _forcesell(bot=MagicMock(), update=update) - - assert rpc_mock.call_count == 4 - for args in rpc_mock.call_args_list: - assert '0.00001098' in args[0][0] - assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0] - assert '-0.089 USD' in args[0][0] - - -def test_forcesell_handle_invalid(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock()) - init(default_conf, create_engine('sqlite://')) - - # Trader is not running - update_state(State.STOPPED) - update.message.text = '/forcesell 1' - _forcesell(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'not running' in msg_mock.call_args_list[0][0][0] - - # No argument - msg_mock.reset_mock() - update_state(State.RUNNING) - update.message.text = '/forcesell' - _forcesell(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'Invalid argument' in msg_mock.call_args_list[0][0][0] - - # Invalid argument - msg_mock.reset_mock() - update_state(State.RUNNING) - update.message.text = '/forcesell 123456' - _forcesell(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'Invalid argument.' in msg_mock.call_args_list[0][0][0] - - -def test_performance_handle( - default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - trade = Trade.query.first() - assert trade - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - - # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) - - trade.close_date = datetime.utcnow() - trade.is_open = False - _performance(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'Performance' in msg_mock.call_args_list[0][0][0] - assert 'BTC_ETH\t6.20% (1)' in msg_mock.call_args_list[0][0][0] - - -def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) trade = Trade.query.first() assert trade @@ -417,7 +403,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_ # Try valid data update.message.text = '/daily 2' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'Daily' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] @@ -429,8 +415,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_ # Reset msg_mock msg_mock.reset_mock() # Add two other trades - create_trade(0.001, int(default_conf['ticker_interval'])) - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) trades = Trade.query.all() for trade in trades: @@ -441,227 +427,178 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_ update.message.text = '/daily 1' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] assert str(' 2.798 USD') in msg_mock.call_args_list[0][0][0] assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] -def test_daily_wrong_input(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: + """ + Test _daily() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) # Try invalid data msg_mock.reset_mock() - update_state(State.RUNNING) + freqtradebot.update_state(State.RUNNING) update.message.text = '/daily -2' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'must be an integer greater than 0' in msg_mock.call_args_list[0][0][0] # Try invalid data msg_mock.reset_mock() - update_state(State.RUNNING) + freqtradebot.update_state(State.RUNNING) update.message.text = '/daily today' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert str('Daily Profit over the last 7 days') in msg_mock.call_args_list[0][0][0] -def test_count_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_profit_handle(default_conf, update, ticker, ticker_sell_up, + limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test _profit() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_order_id')) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - _count(bot=MagicMock(), update=update) + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._profit(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'not running' in msg_mock.call_args_list[0][0][0] + assert 'no closed trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - update_state(State.RUNNING) # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + trade = Trade.query.first() + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + telegram._profit(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'no closed trade' in msg_mock.call_args_list[-1][0][0] msg_mock.reset_mock() - _count(bot=MagicMock(), update=update) - msg = '
  current    max\n---------  -----\n        1      {}
'.format( - default_conf['max_open_trades'] - ) - assert msg in msg_mock.call_args_list[0][0][0] + # Update the ticker with a market going up + mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', ticker_sell_up) + trade.update(limit_sell_order) + trade.close_date = datetime.utcnow() + trade.is_open = False -def test_performance_handle_invalid(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock()) - init(default_conf, create_engine('sqlite://')) - - # Trader is not running - update_state(State.STOPPED) - _performance(bot=MagicMock(), update=update) + telegram._profit(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'not running' in msg_mock.call_args_list[0][0][0] + assert '*ROI:* Close trades' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + + assert '*Best Performing:* `BTC_ETH: 6.20%`' in msg_mock.call_args_list[-1][0][0] -def test_start_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - assert get_state() == State.STOPPED - _start(bot=MagicMock(), update=update) - assert get_state() == State.RUNNING - assert msg_mock.call_count == 0 - - -def test_start_handle_already_running(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.RUNNING) - assert get_state() == State.RUNNING - _start(bot=MagicMock(), update=update) - assert get_state() == State.RUNNING - assert msg_mock.call_count == 1 - assert 'already running' in msg_mock.call_args_list[0][0][0] - - -def test_stop_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.RUNNING) - assert get_state() == State.RUNNING - _stop(bot=MagicMock(), update=update) - assert get_state() == State.STOPPED - assert msg_mock.call_count == 1 - assert 'Stopping trader' in msg_mock.call_args_list[0][0][0] - - -def test_stop_handle_already_stopped(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - assert get_state() == State.STOPPED - _stop(bot=MagicMock(), update=update) - assert get_state() == State.STOPPED - assert msg_mock.call_count == 1 - assert 'already stopped' in msg_mock.call_args_list[0][0][0] - - -def test_telegram_balance_handle(default_conf, update, mocker): - mock_balance = [{ - 'Currency': 'BTC', - 'Balance': 10.0, - 'Available': 12.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'ETH', - 'Balance': 0.0, - 'Available': 0.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'USDT', - 'Balance': 10000.0, - 'Available': 0.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'LTC', - 'Balance': 10.0, - 'Available': 10.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }] +def test_telegram_balance_handle(default_conf, update, mocker) -> None: + """ + Test _balance() method + """ + mock_balance = [ + { + 'Currency': 'BTC', + 'Balance': 10.0, + 'Available': 12.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'ETH', + 'Balance': 0.0, + 'Available': 0.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'USDT', + 'Balance': 10000.0, + 'Available': 0.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'LTC', + 'Balance': 10.0, + 'Available': 10.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + } + ] def mock_ticker(symbol, refresh): + """ + Mock Bittrex.get_ticker() response + """ if symbol == 'USDT_BTC': return { 'bid': 10000.00, 'ask': 10000.00, 'last': 10000.00, } - else: - return { - 'bid': 0.1, - 'ask': 0.1, - 'last': 0.1, - } - mocker.patch.dict('freqtrade.main._CONF', default_conf) + return { + 'bid': 0.1, + 'ask': 0.1, + 'last': 0.1, + } + + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=mock_balance) + mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', side_effect=mock_ticker) + msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - get_balances=MagicMock(return_value=mock_balance)) - mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', - ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1})) - mocker.patch('freqtrade.main.exchange.get_ticker', side_effect=mock_ticker) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) - _balance(bot=MagicMock(), update=update) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._balance(bot=MagicMock(), update=update) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert '*Currency*: BTC' in result @@ -672,135 +609,462 @@ def test_telegram_balance_handle(default_conf, update, mocker): assert '*BTC*: 12.00000000' in result -def test_zero_balance_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_zero_balance_handle(default_conf, update, mocker) -> None: + """ + Test _balance() method when the Exchange platform returns nothing + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=[]) + msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - get_balances=MagicMock(return_value=[])) - _balance(bot=MagicMock(), update=update) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._balance(bot=MagicMock(), update=update) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert '`All balances are zero.`' in result -def test_help_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_start_handle(default_conf, update, mocker) -> None: + """ + Test _start() method + """ + patch_pymarketcap(mocker) msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - _help(bot=MagicMock(), update=update) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + assert freqtradebot.get_state() == State.STOPPED + telegram._start(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.RUNNING + assert msg_mock.call_count == 0 + + +def test_start_handle_already_running(default_conf, update, mocker) -> None: + """ + Test _start() method + """ + patch_pymarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.RUNNING) + assert freqtradebot.get_state() == State.RUNNING + telegram._start(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.RUNNING + assert msg_mock.call_count == 1 + assert 'already running' in msg_mock.call_args_list[0][0][0] + + +def test_stop_handle(default_conf, update, mocker) -> None: + """ + Test _stop() method + """ + patch_pymarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.RUNNING) + assert freqtradebot.get_state() == State.RUNNING + telegram._stop(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.STOPPED + assert msg_mock.call_count == 1 + assert 'Stopping trader' in msg_mock.call_args_list[0][0][0] + + +def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: + """ + Test _stop() method + """ + patch_pymarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + assert freqtradebot.get_state() == State.STOPPED + telegram._stop(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.STOPPED + assert msg_mock.call_count == 1 + assert 'already stopped' in msg_mock.call_args_list[0][0][0] + + +def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', ticker_sell_up) + + update.message.text = '/forcesell 1' + telegram._forcesell(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down + ) + + trade = Trade.query.first() + assert trade + + update.message.text = '/forcesell 1' + telegram._forcesell(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_forcesell_all_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + for _ in range(4): + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + rpc_mock.reset_mock() + + update.message.text = '/forcesell all' + telegram._forcesell(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 4 + for args in rpc_mock.call_args_list: + assert '0.00001098' in args[0][0] + assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0] + assert '-0.089 USD' in args[0][0] + + +def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Trader is not running + freqtradebot.update_state(State.STOPPED) + update.message.text = '/forcesell 1' + telegram._forcesell(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'not running' in msg_mock.call_args_list[0][0][0] + + # No argument + msg_mock.reset_mock() + freqtradebot.update_state(State.RUNNING) + update.message.text = '/forcesell' + telegram._forcesell(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'Invalid argument' in msg_mock.call_args_list[0][0][0] + + # Invalid argument + msg_mock.reset_mock() + freqtradebot.update_state(State.RUNNING) + update.message.text = '/forcesell 123456' + telegram._forcesell(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'Invalid argument.' in msg_mock.call_args_list[0][0][0] + + +def test_performance_handle(default_conf, update, ticker, limit_buy_order, + limit_sell_order, mocker) -> None: + """ + Test _performance() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + trade = Trade.query.first() + assert trade + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + # Simulate fulfilled LIMIT_SELL order for trade + trade.update(limit_sell_order) + + trade.close_date = datetime.utcnow() + trade.is_open = False + telegram._performance(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'Performance' in msg_mock.call_args_list[0][0][0] + assert 'BTC_ETH\t6.20% (1)' in msg_mock.call_args_list[0][0][0] + + +def test_performance_handle_invalid(default_conf, update, mocker) -> None: + """ + Test _performance() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Trader is not running + freqtradebot.update_state(State.STOPPED) + telegram._performance(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'not running' in msg_mock.call_args_list[0][0][0] + + +def test_count_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _count() method + """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_order_id') + ) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + telegram._count(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'not running' in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + freqtradebot.update_state(State.RUNNING) + + # Create some test data + freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + msg_mock.reset_mock() + telegram._count(bot=MagicMock(), update=update) + + msg = '
  current    max\n---------  -----\n        1      {}
'.format( + default_conf['max_open_trades'] + ) + assert msg in msg_mock.call_args_list[0][0][0] + + +def test_help_handle(default_conf, update, mocker) -> None: + """ + Test _help() method + """ + patch_pymarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._help(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert '*/help:* `This help message`' in msg_mock.call_args_list[0][0][0] -def test_version_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_version_handle(default_conf, update, mocker) -> None: + """ + Test _version() method + """ + patch_pymarketcap(mocker) msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) - _version(bot=MagicMock(), update=update) + telegram._version(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert '*Version:* `{}`'.format(__version__) in msg_mock.call_args_list[0][0][0] -def test_send_msg(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) +def test_send_msg(default_conf, mocker) -> None: + """ + Test send_msg() method + """ + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + conf = deepcopy(default_conf) bot = MagicMock() - send_msg('test', bot) + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._config['telegram']['enabled'] = False + telegram.send_msg('test', bot) assert not bot.method_calls bot.reset_mock() - default_conf['telegram']['enabled'] = True - send_msg('test', bot) + telegram._config['telegram']['enabled'] = True + telegram.send_msg('test', bot) assert len(bot.method_calls) == 1 -def test_send_msg_network_error(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - default_conf['telegram']['enabled'] = True +def test_send_msg_network_error(default_conf, mocker, caplog) -> None: + """ + Test send_msg() method + """ + patch_pymarketcap(mocker) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + conf = deepcopy(default_conf) bot = MagicMock() bot.send_message = MagicMock(side_effect=NetworkError('Oh snap')) - send_msg('test', bot) + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._config['telegram']['enabled'] = True + telegram.send_msg('test', bot) # Bot should've tried to send it twice assert len(bot.method_calls) == 2 - - -def test_init(default_conf, mocker): - start_polling = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - # mock telegram.ext.Updater - Updater=MagicMock(return_value=start_polling)) - # not enabled - tg.init(default_conf) - assert start_polling.call_count == 0 - # number of handles registered - assert start_polling.dispatcher.add_handler.call_count == 11 - assert start_polling.start_polling.call_count == 1 - - # enabled - default_conf['telegram'] = {} - default_conf['telegram']['enabled'] = True - default_conf['telegram']['token'] = '' - tg.init(default_conf) - - -def test_cleanup(default_conf, mocker): - default_conf['telegram'] = {} - default_conf['telegram']['enabled'] = False - updater_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - _UPDATER=updater_mock) - # not enabled - tg.cleanup() - assert updater_mock.stop.call_count == 0 - - # enabled - default_conf['telegram']['enabled'] = True - tg.cleanup() - assert updater_mock.stop.call_count == 1 - - -def test_status(default_conf, update, mocker): - update.message.chat.id = 123 - default_conf['telegram'] = {} - default_conf['telegram']['chat_id'] = 123 - mocker.patch('telegram.update', MagicMock()) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - msg_mock = MagicMock() - status_table = MagicMock() - mocker.patch.multiple('freqtrade.rpc', - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), - _status_table=status_table, - send_msg=msg_mock) - _status(bot=MagicMock(), update=update) - assert msg_mock.call_count == 3 - update.message.text = MagicMock() - update.message.text.replace = MagicMock(return_value='table 2 3') - _status(bot=MagicMock(), update=update) - assert status_table.call_count == 1 + assert tt.log_has( + 'Got TelegramError: Oh snap! Giving up on that message.', + caplog.record_tuples + ) From 6ef7b7d93d6764ec33d1dbc17678824a0dd4da6e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 2 Mar 2018 21:46:32 +0800 Subject: [PATCH 13/56] Complete Backtesting and Hyperopt unit tests --- freqtrade/analyze.py | 2 +- freqtrade/configuration.py | 35 +- freqtrade/logger.py | 5 +- freqtrade/optimize/backtesting.py | 13 +- freqtrade/optimize/hyperopt.py | 1002 +++++++++--------- freqtrade/tests/optimize/test_backtesting.py | 34 +- freqtrade/tests/optimize/test_hyperopt.py | 373 ++++--- freqtrade/tests/optimize/test_optimize.py | 11 - freqtrade/tests/test_analyze.py | 16 +- freqtrade/tests/test_dataframe.py | 8 +- 10 files changed, 851 insertions(+), 648 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index f3126d723..bde0c9f80 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -30,7 +30,7 @@ class Analyze(object): Init Analyze :param config: Bot configuration (use the one from Configuration()) """ - self.logger = Logger(name=__name__).get_logger() + self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() self.config = config self.strategy = Strategy(self.config) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 38925bd4e..721def2a4 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -19,7 +19,8 @@ class Configuration(object): """ def __init__(self, args: List[str]) -> None: self.args = args - self.logger = Logger(name=__name__).get_logger() + self.logging = Logger(name=__name__) + self.logger = self.logging.get_logger() self.config = self._load_config() self.show_info() @@ -35,16 +36,24 @@ class Configuration(object): config.update({'strategy': self.args.strategy}) # Add dynamic_whitelist if found - if self.args.dynamic_whitelist: + if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) # Add dry_run_db if found and the bot in dry run if self.args.dry_run_db and config.get('dry_run', False): config.update({'dry_run_db': True}) - # Load Backtesting / Hyperopt + # Log level + if 'loglevel' in self.args and self.args.loglevel: + config.update({'loglevel': self.args.loglevel}) + self.logging.set_level(self.args.loglevel) + + # Load Backtesting config = self._load_backtesting_config(config) + # Load Hyperopt + config = self._load_hyperopt_config(config) + return config def _load_config_file(self, path: str) -> Dict[str, Any]: @@ -64,7 +73,7 @@ class Configuration(object): def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ - Extract information for sys.argv and load Backtesting and Hyperopt configuration + Extract information for sys.argv and load Backtesting configuration :return: configuration as dictionary """ # If -i/--ticker-interval is used we override the configuration parameter @@ -107,6 +116,24 @@ class Configuration(object): return config + def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load Hyperopt configuration + :return: configuration as dictionary + """ + # If --realistic-simulation is used we add it to the configuration + if 'epochs' in self.args and self.args.epochs: + config.update({'epochs': self.args.epochs}) + self.logger.info('Parameter --epochs detected ...') + self.logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) + + # If --mongodb is used we add it to the configuration + if 'mongodb' in self.args and self.args.mongodb: + config.update({'mongodb': self.args.mongodb}) + self.logger.info('Parameter --use-mongodb detected ...') + + return config + def _validate_config(self, conf: Dict[str, Any]) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema diff --git a/freqtrade/logger.py b/freqtrade/logger.py index 83bbbfd0d..95e55e477 100644 --- a/freqtrade/logger.py +++ b/freqtrade/logger.py @@ -19,9 +19,12 @@ class Logger(object): :return: None """ self.name = name - self.level = level self.logger = None + if level is None: + level = logging.INFO + self.level = level + self._init_logger() def _init_logger(self) -> None: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 60b014872..c15aee1fd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -5,7 +5,6 @@ This module contains the backtesting logic """ from typing import Dict, Tuple, Any -import logging import arrow from pandas import DataFrame, Series from tabulate import tabulate @@ -20,6 +19,7 @@ from freqtrade.logger import Logger from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +from memory_profiler import profile class Backtesting(object): """ @@ -30,7 +30,9 @@ class Backtesting(object): backtesting.start() """ def __init__(self, config: Dict[str, Any]) -> None: - self.logging = Logger(name=__name__) + + # Init the logger + self.logging = Logger(name=__name__, level=config['loglevel']) self.logger = self.logging.get_logger() self.config = config @@ -219,6 +221,7 @@ class Backtesting(object): labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) + @profile(precision=10) def start(self) -> None: """ Run a backtesting end-to-end @@ -246,10 +249,14 @@ class Backtesting(object): ) max_open_trades = self.config.get('max_open_trades', 0) - preprocessed = self.tickerdata_to_dataframe(data) + # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) + + import pprint + pprint.pprint(min_date) + pprint.pprint(max_date) self.logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8b89e1985..c1d95d6ef 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -1,5 +1,8 @@ -# pragma pylint: disable=missing-docstring,W0212,W0603 +# pragma pylint: disable=too-many-instance-attributes, pointless-string-statement +""" +This module contains the hyperopt logic +""" import json import logging @@ -19,521 +22,574 @@ from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -# Monkey patch config -from freqtrade import main # noqa; noqa -from freqtrade import exchange, misc, optimize -from freqtrade.exchange import Bittrex -from freqtrade.misc import load_config -from freqtrade.optimize import backtesting -from freqtrade.optimize.backtesting import backtest -from freqtrade.strategy.strategy import Strategy +from freqtrade.configuration import Configuration +from freqtrade.optimize import load_data +from freqtrade.arguments import Arguments +from freqtrade.optimize.backtesting import Backtesting, setup_configuration +from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf -# Remove noisy log messages -logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) -logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) -logger = logging.getLogger(__name__) - -# set TARGET_TRADES to suit your number concurrent trades so its realistic to the number of days -TARGET_TRADES = 600 -TOTAL_TRIES = 0 -_CURRENT_TRIES = 0 -CURRENT_BEST_LOSS = 100 - -# max average trade duration in minutes -# if eval ends with higher value, we consider it a failed eval -MAX_ACCEPTED_TRADE_DURATION = 300 - -# this is expexted avg profit * expected trade count -# for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85 -# check that the reported Σ% values do not exceed this! -EXPECTED_MAX_PROFIT = 3.0 - -# Configuration and data used by hyperopt -PROCESSED = None # optimize.preprocess(optimize.load_data()) -OPTIMIZE_CONFIG = hyperopt_optimize_conf() - -# Hyperopt Trials -TRIALS_FILE = os.path.join('user_data', 'hyperopt_trials.pickle') -TRIALS = Trials() - -main._CONF = OPTIMIZE_CONFIG - - -def populate_indicators(dataframe: DataFrame) -> DataFrame: +class Hyperopt(Backtesting): """ - Adds several different TA indicators to the given DataFrame + Hyperopt class, this class contains all the logic to run a hyperopt simulation + + To run a backtest: + hyperopt = Hyperopt(config) + hyperopt.start() """ - dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] + def __init__(self, config: Dict[str, Any]) -> None: - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ + super().__init__(config) - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - - return dataframe + # Rename the logging to display Hyperopt file instead of Backtesting + self.logging = Logger(name=__name__, level=config['loglevel']) + self.logger = self.logging.get_logger() -def save_trials(trials, trials_path=TRIALS_FILE): - """Save hyperopt trials to file""" - logger.info('Saving Trials to \'{}\''.format(trials_path)) - pickle.dump(trials, open(trials_path, 'wb')) + # set TARGET_TRADES to suit your number concurrent trades so its realistic + # to the number of days + self.target_trades = 600 + self.total_tries = config.get('epochs', 0) + self.current_tries = 0 + self.current_best_loss = 100 + # max average trade duration in minutes + # if eval ends with higher value, we consider it a failed eval + self.max_accepted_trade_duration = 300 -def read_trials(trials_path=TRIALS_FILE): - """Read hyperopt trials file""" - logger.info('Reading Trials from \'{}\''.format(trials_path)) - trials = pickle.load(open(trials_path, 'rb')) - os.remove(trials_path) - return trials + # this is expexted avg profit * expected trade count + # for example 3.5%, 1100 trades, self.expected_max_profit = 3.85 + # check that the reported Σ% values do not exceed this! + self.expected_max_profit = 3.0 + # Configuration and data used by hyperopt + self.processed = None -def log_trials_result(trials): - vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) - results = trials.best_trial['result']['result'] - logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + # Hyperopt Trials + self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials = Trials() + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] -def log_results(results): - """ log results if it is better than any previous evaluation """ - global CURRENT_BEST_LOSS + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ - if results['loss'] < CURRENT_BEST_LOSS: - CURRENT_BEST_LOSS = results['loss'] - logger.info('{:5d}/{}: {}. Loss {:.5f}'.format( - results['current_tries'], - results['total_tries'], - results['result'], - results['loss'])) - else: - print('.', end='') - sys.stdout.flush() + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ -def calculate_loss(total_profit: float, trade_count: int, trade_duration: float): - """ objective function, returns smaller number for more optimal results """ - trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8) - profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) - return trade_loss + profit_loss + duration_loss - - -def generate_roi_table(params) -> Dict[str, float]: - roi_table = {} - roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2'] - roi_table[str(params['roi_t3'] + params['roi_t2'])] = params['roi_p1'] - roi_table[str(params['roi_t3'] + params['roi_t2'] + params['roi_t1'])] = 0 - - return roi_table - - -def roi_space() -> Dict[str, Any]: - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } - - -def stoploss_space() -> Dict[str, Any]: - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } - - -def indicator_space() -> Dict[str, Any]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), - } - - -def hyperopt_space() -> Dict[str, Any]: - return {**indicator_space(), **roi_space(), **stoploss_space()} - - -def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) - - # TRIGGERS - triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), - } - conditions.append(triggers.get(params['trigger']['type'])) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] return dataframe - return populate_buy_trend + def save_trials(self) -> None: + """ + Save hyperopt trials to file + """ + self.logger.info('Saving Trials to \'%s\'', self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) + def read_trials(self) -> Trials: + """ + Read hyperopt trials file + """ + self.logger.info('Reading Trials from \'%s\'', self.trials_file) + trials = pickle.load(open(self.trials_file, 'rb')) + os.remove(self.trials_file) + return trials -def optimizer(params): - global _CURRENT_TRIES + def log_trials_result(self) -> None: + """ + Display Best hyperopt result + """ + vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) + results = self.trials.best_trial['result']['result'] + self.logger.info('Best result:\n%s\nwith values:\n%s', results, vals) - if 'roi_t1' in params: - strategy = Strategy() - strategy.minimal_roi = generate_roi_table(params) + def log_results(self, results) -> None: + """ + Log results if it is better than any previous evaluation + """ + if results['loss'] < self.current_best_loss: + self.current_best_loss = results['loss'] + log_msg = '{:5d}/{}: {}. Loss {:.5f}'.format( + results['current_tries'], + results['total_tries'], + results['result'], + results['loss'] + ) + self.logger.info(log_msg) + else: + print('.', end='') + sys.stdout.flush() - backtesting.populate_buy_trend = buy_strategy_generator(params) + def calculate_loss(self, total_profit: float, trade_count: int, trade_duration: float) -> float: + """ + Objective function, returns smaller number for more optimal results + """ + trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) + profit_loss = max(0, 1 - total_profit / self.expected_max_profit) + duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) + return trade_loss + profit_loss + duration_loss - results = backtest({'stake_amount': OPTIMIZE_CONFIG['stake_amount'], - 'processed': PROCESSED, - 'stoploss': params['stoploss']}) - result_explanation = format_results(results) + @staticmethod + def generate_roi_table(params) -> Dict[str, float]: + """ + Generate the ROI table thqt will be used by Hyperopt + """ + roi_table = {} + roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2'] + roi_table[str(params['roi_t3'] + params['roi_t2'])] = params['roi_p1'] + roi_table[str(params['roi_t3'] + params['roi_t2'] + params['roi_t1'])] = 0 - total_profit = results.profit_percent.sum() - trade_count = len(results.index) - trade_duration = results.duration.mean() * 5 + return roi_table - if trade_count == 0 or trade_duration > MAX_ACCEPTED_TRADE_DURATION: - print('.', end='') + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ return { - 'status': STATUS_FAIL, - 'loss': float('inf') + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), } - loss = calculate_loss(total_profit, trade_count, trade_duration) + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } - _CURRENT_TRIES += 1 + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } - log_results({ - 'loss': loss, - 'current_tries': _CURRENT_TRIES, - 'total_tries': TOTAL_TRIES, - 'result': result_explanation, - }) + @staticmethod + def hyperopt_space() -> Dict[str, Any]: + """ + Return the space to use during Hyperopt + """ + return { + **Hyperopt.indicator_space(), + **Hyperopt.roi_space(), + **Hyperopt.stoploss_space() + } - return { - 'loss': loss, - 'status': STATUS_OK, - 'result': result_explanation, - } + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) -def format_results(results: DataFrame): - return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.profit_percent.sum(), - results.duration.mean() * 5, + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + def optimizer(self, params) -> Dict: + if 'roi_t1' in params: + self.analyze.strategy.minimal_roi = self.generate_roi_table(params) + + self.populate_buy_trend = self.buy_strategy_generator(params) + + results = self.backtest( + { + 'stake_amount': self.config['stake_amount'], + 'processed': self.processed, + 'stoploss': params['stoploss'] + } + ) + result_explanation = self.format_results(results) + + total_profit = results.profit_percent.sum() + trade_count = len(results.index) + trade_duration = results.duration.mean() * 5 + + if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: + print('.', end='') + return { + 'status': STATUS_FAIL, + 'loss': float('inf') + } + + loss = self.calculate_loss(total_profit, trade_count, trade_duration) + + self.current_tries += 1 + + self.log_results( + { + 'loss': loss, + 'current_tries': self.current_tries, + 'total_tries': self.total_tries, + 'result': result_explanation, + } + ) + + return { + 'loss': loss, + 'status': STATUS_OK, + 'result': result_explanation, + } + + @staticmethod + def format_results(results: DataFrame) -> str: + """ + Return the format result in a string + """ + return ('{:6d} trades. Avg profit {: 5.2f}%. ' + 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.profit_percent.sum(), + results.duration.mean() * 5, + ) + + def start(self): + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = load_data( + datadir=self.config.get('datadir'), + pairs=self.config['exchange']['pair_whitelist'], + ticker_interval=self.ticker_interval, + timerange=timerange + ) + + self.analyze.populate_indicators = Hyperopt.populate_indicators + self.processed = self.tickerdata_to_dataframe(data) + + if self.config.get('mongodb'): + self.logger.info('Using mongodb ...') + self.logger.info( + 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' ) + db_name = 'freqtrade_hyperopt' + self.trials = MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) + else: + self.logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file): + self.trials = self.read_trials() -def start(args): - global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries + self.logger.info( + 'Continuing with trials. Current: {}, Total: {}' + .format(self.current_tries, self.total_tries) + ) - TOTAL_TRIES = args.epochs + try: + # change the Logging format + self.logging.set_format('\n%(message)s') - exchange._API = Bittrex({'key': '', 'secret': ''}) + best_parameters = fmin( + fn=self.optimizer, + space=self.hyperopt_space(), + algo=tpe.suggest, + max_evals=self.total_tries, + trials=self.trials + ) + + results = sorted(self.trials.results, key=itemgetter('loss')) + best_result = results[0]['result'] + + except ValueError: + best_parameters = {} + best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ + 'try with more epochs (param: -e).' + + # Improve best parameter logging display + if best_parameters: + best_parameters = space_eval( + self.hyperopt_space(), + best_parameters + ) + + self.logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + if 'roi_t1' in best_parameters: + self.logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + + self.logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + self.save_trials() + + def signal_handler(self, sig, frame): + """ + Hyperopt SIGINT handler + """ + self.logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + + self.save_trials() + self.log_trials_result() + sys.exit(0) + + +def start(args) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + + # Remove noisy log messages + logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) # Initialize logger - logging.basicConfig( - level=args.loglevel, - format='\n%(message)s', - ) + logger = Logger(name=__name__).get_logger() + logger.info('Starting freqtrade in Hyperopt mode') - logger.info('Using config: %s ...', args.config) - config = load_config(args.config) - pairs = config['exchange']['pair_whitelist'] + # Initialize configuration + #config = setup_configuration(args) - # If -i/--ticker-interval is use we override the configuration parameter - # (that will override the strategy configuration) - if args.ticker_interval: - config.update({'ticker_interval': args.ticker_interval}) + # Monkey patch of the configuration with hyperopt_conf.py + configuration = Configuration(args) + optimize_config = hyperopt_optimize_conf() + config = configuration._load_backtesting_config(optimize_config) + config = configuration._load_hyperopt_config(config) + config['exchange']['key'] = '' + config['exchange']['secret'] = '' - # init the strategy to use - config.update({'strategy': args.strategy}) - strategy = Strategy() - strategy.init(config) - - timerange = misc.parse_timerange(args.timerange) - data = optimize.load_data(args.datadir, pairs=pairs, - ticker_interval=strategy.ticker_interval, - timerange=timerange) - optimize.populate_indicators = populate_indicators - PROCESSED = optimize.tickerdata_to_dataframe(data) - - if args.mongodb: - logger.info('Using mongodb ...') - logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') - - db_name = 'freqtrade_hyperopt' - TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') - else: - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, signal_handler) - # read trials file if we have one - if os.path.exists(TRIALS_FILE): - TRIALS = read_trials() - - _CURRENT_TRIES = len(TRIALS.results) - TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES - logger.info( - 'Continuing with trials. Current: {}, Total: {}' - .format(_CURRENT_TRIES, TOTAL_TRIES)) - - try: - best_parameters = fmin( - fn=optimizer, - space=hyperopt_space(), - algo=tpe.suggest, - max_evals=TOTAL_TRIES, - trials=TRIALS - ) - - results = sorted(TRIALS.results, key=itemgetter('loss')) - best_result = results[0]['result'] - - except ValueError: - best_parameters = {} - best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ - 'try with more epochs (param: -e).' - - # Improve best parameter logging display - if best_parameters: - best_parameters = space_eval( - hyperopt_space(), - best_parameters - ) - - logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - if 'roi_t1' in best_parameters: - logger.info('ROI table:\n%s', generate_roi_table(best_parameters)) - logger.info('Best Result:\n%s', best_result) - - # Store trials result to file to resume next time - save_trials(TRIALS) - - -def signal_handler(sig, frame): - """Hyperopt SIGINT handler""" - logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) - - save_trials(TRIALS) - log_trials_result(TRIALS) - sys.exit(0) + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 7a0158162..d4f705c51 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -5,6 +5,7 @@ import math from typing import List from copy import deepcopy from unittest.mock import MagicMock +from arrow import Arrow import pandas as pd from freqtrade import optimize from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration @@ -255,6 +256,25 @@ def test_backtesting_init(default_conf) -> None: assert callable(backtesting.populate_sell_trend) +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Backtesting.tickerdata_to_dataframe() method + """ + + timerange = ((None, 'line'), None, -100) + tick = optimize.load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) + tickerlist = {'BTC_UNITEST': tick} + + backtesting = _BACKTESTING + data = backtesting.tickerdata_to_dataframe(tickerlist) + assert len(data['BTC_UNITEST']) == 100 + + # Load Analyze to compare the result between Backtesting function and Analyze are the same + analyze = Analyze(default_conf) + data2 = analyze.tickerdata_to_dataframe(tickerlist) + assert data['BTC_UNITEST'].equals(data2['BTC_UNITEST']) + + def test_get_timeframe() -> None: """ Test Backtesting.get_timeframe() method @@ -308,8 +328,18 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: """ Test Backtesting.start() method """ - mocker.patch.multiple('freqtrade.optimize', load_data=mocked_load_data) - mocker.patch('freqtrade.exchange.get_ticker_history', MagicMock) + def get_timeframe(input1, input2): + return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.exchange.get_ticker_history') + mocker.patch.multiple( + 'freqtrade.optimize.backtesting.Backtesting', + backtest=MagicMock(), + _generate_text_table=MagicMock(return_value='1'), + get_timeframe=get_timeframe, + ) conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f127ac8fd..63e4404ff 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,117 +1,108 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import logging +import os +import pytest +from copy import deepcopy -from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results, save_trials, read_trials, generate_roi_table +#from freqtrade.optimize.hyperopt import EXPECTED_MAX_PROFIT, start, \ +# log_results, save_trials, read_trials, generate_roi_table +from unittest.mock import MagicMock + +from freqtrade.optimize.hyperopt import Hyperopt, start +import freqtrade.tests.conftest as tt # test tools -def test_loss_calculation_prefer_correct_trade_count(): - correct = calculate_loss(1, TARGET_TRADES, 20) - over = calculate_loss(1, TARGET_TRADES + 100, 20) - under = calculate_loss(1, TARGET_TRADES - 100, 20) - assert over > correct - assert under > correct +# Avoid to reinit the same object again and again +_HYPEROPT = Hyperopt(tt.default_conf()) -def test_loss_calculation_prefer_shorter_trades(): - shorter = calculate_loss(1, 100, 20) - longer = calculate_loss(1, 100, 30) - assert shorter < longer - - -def test_loss_calculation_has_limited_profit(): - correct = calculate_loss(EXPECTED_MAX_PROFIT, TARGET_TRADES, 20) - over = calculate_loss(EXPECTED_MAX_PROFIT * 2, TARGET_TRADES, 20) - under = calculate_loss(EXPECTED_MAX_PROFIT / 2, TARGET_TRADES, 20) - assert over == correct - assert under > correct - - -def create_trials(mocker): +# Functions for recurrent object patching +def create_trials(mocker) -> None: """ When creating trials, mock the hyperopt Trials so that *by default* - we don't create any pickle'd files in the filesystem - we might have a pickle'd file so make sure that we return false when looking for it """ - mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', - return_value='freqtrade/tests/optimize/ut_trials.pickle') - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', - return_value=False) - mocker.patch('freqtrade.optimize.hyperopt.save_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.read_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.os.remove', - return_value=True) + _HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize','ut_trials.pickle') + + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + return mocker.Mock( - results=[{ - 'loss': 1, - 'result': 'foo', - 'status': 'ok' - }], + results=[ + { + 'loss': 1, + 'result': 'foo', + 'status': 'ok' + } + ], best_trial={'misc': {'vals': {'adx': 999}}} ) -def test_start_calls_fmin(mocker): - trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) - mocker.patch('freqtrade.optimize.hyperopt.sorted', - return_value=trials.results) - mocker.patch('freqtrade.optimize.preprocess') - mocker.patch('freqtrade.optimize.load_data') - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) +# Unit tests +def test_loss_calculation_prefer_correct_trade_count() -> None: + """ + Test Hyperopt.calculate_loss() + """ + hyperopt = _HYPEROPT - args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False, - timerange=None) - start(args) - - mock_fmin.assert_called_once() + correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) + over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20) + under = hyperopt.calculate_loss(1, hyperopt.target_trades - 100, 20) + assert over > correct + assert under > correct -def test_start_uses_mongotrials(mocker): - mock_mongotrials = mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', - return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) +def test_loss_calculation_prefer_shorter_trades() -> None: + """ + Test Hyperopt.calculate_loss() + """ + hyperopt = _HYPEROPT - args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True, - timerange=None) - start(args) - - mock_mongotrials.assert_called_once() + shorter = hyperopt.calculate_loss(1, 100, 20) + longer = hyperopt.calculate_loss(1, 100, 30) + assert shorter < longer -def test_log_results_if_loss_improves(mocker): - logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') - global CURRENT_BEST_LOSS - CURRENT_BEST_LOSS = 2 - log_results({ - 'loss': 1, - 'current_tries': 1, - 'total_tries': 2, - 'result': 'foo' - }) +def test_loss_calculation_has_limited_profit() -> None: + hyperopt = _HYPEROPT - logger.assert_called_once() + correct = hyperopt.calculate_loss(hyperopt.expected_max_profit, hyperopt.target_trades, 20) + over = hyperopt.calculate_loss(hyperopt.expected_max_profit * 2, hyperopt.target_trades, 20) + under = hyperopt.calculate_loss(hyperopt.expected_max_profit / 2, hyperopt.target_trades, 20) + assert over == correct + assert under > correct -def test_no_log_if_loss_does_not_improve(mocker): - logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') - global CURRENT_BEST_LOSS - CURRENT_BEST_LOSS = 2 - log_results({ - 'loss': 3, - }) - - assert not logger.called +def test_log_results_if_loss_improves(caplog) -> None: + hyperopt = _HYPEROPT + hyperopt.current_best_loss = 2 + hyperopt.log_results( + { + 'loss': 1, + 'current_tries': 1, + 'total_tries': 2, + 'result': 'foo' + } + ) + assert tt.log_has(' 1/2: foo. Loss 1.00000', caplog.record_tuples) -def test_fmin_best_results(mocker, caplog): - caplog.set_level(logging.INFO) +def test_no_log_if_loss_does_not_improve(caplog) -> None: + hyperopt = _HYPEROPT + hyperopt.current_best_loss = 2 + hyperopt.log_results( + { + 'loss': 3, + } + ) + assert caplog.record_tuples == [] + + +def test_fmin_best_results(mocker, default_conf, caplog) -> None: fmin_result = { "macd_below_zero": 0, "adx": 1, @@ -136,38 +127,65 @@ def test_fmin_best_results(mocker, caplog): "roi_p3": 3, } - mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'timerange': None}) - args = mocker.Mock(epochs=1, config='config.json.example', - timerange=None) - start(args) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + hyperopt = Hyperopt(conf) + hyperopt.trials = create_trials(mocker) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() exists = [ - 'Best parameters', + 'Best parameters:', '"adx": {\n "enabled": true,\n "value": 15.0\n },', + '"fastd": {\n "enabled": true,\n "value": 40.0\n },', '"green_candle": {\n "enabled": true\n },', + '"macd_below_zero": {\n "enabled": false\n },', '"mfi": {\n "enabled": false\n },', + '"over_sar": {\n "enabled": false\n },', + '"roi_p1": 1.0,', + '"roi_p2": 2.0,', + '"roi_p3": 3.0,', + '"roi_t1": 1.0,', + '"roi_t2": 2.0,', + '"roi_t3": 3.0,', + '"rsi": {\n "enabled": true,\n "value": 37.0\n },', + '"stoploss": -0.1,', '"trigger": {\n "type": "faststoch10"\n },', - '"stoploss": -0.1', + '"uptrend_long_ema": {\n "enabled": true\n },', + '"uptrend_short_ema": {\n "enabled": false\n },', + '"uptrend_sma": {\n "enabled": false\n }', + 'ROI table:\n{\'0\': 6.0, \'3.0\': 3.0, \'5.0\': 1.0, \'6.0\': 0}', + 'Best Result:\nfoo' ] - for line in exists: assert line in caplog.text -def test_fmin_throw_value_error(mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') +def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None: + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) - args = mocker.Mock(epochs=1, config='config.json.example', - timerange=None) - start(args) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'timerange': None}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + hyperopt = Hyperopt(conf) + hyperopt.trials = create_trials(mocker) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() exists = [ 'Best Result:', @@ -179,68 +197,80 @@ def test_fmin_throw_value_error(mocker, caplog): assert line in caplog.text -def test_resuming_previous_hyperopt_results_succeeds(mocker): - import freqtrade.optimize.hyperopt as hyperopt +def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> None: trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.hyperopt.TRIALS', - return_value=trials) - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', - return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.len', - return_value=len(trials.results)) - mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', - return_value=trials) - mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.sorted', - return_value=trials.results) - mocker.patch('freqtrade.optimize.preprocess') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', - return_value={}) - args = mocker.Mock(epochs=1, - config='config.json.example', - mongodb=False, - timerange=None) - start(args) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': False}) + conf.update({'timerange': None}) + + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results)) + mock_read = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.read_trials', + return_value=trials + ) + mock_save = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.save_trials', + return_value=None + ) + mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + hyperopt = Hyperopt(conf) + hyperopt.trials = trials + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() mock_read.assert_called_once() mock_save.assert_called_once() - current_tries = hyperopt._CURRENT_TRIES - total_tries = hyperopt.TOTAL_TRIES + current_tries = hyperopt.current_tries + total_tries = hyperopt.total_tries assert current_tries == len(trials.results) assert total_tries == (current_tries + len(trials.results)) -def test_save_trials_saves_trials(mocker): +def test_save_trials_saves_trials(mocker, caplog) -> None: + create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + + hyperopt = _HYPEROPT + mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) + + hyperopt.save_trials() + + assert tt.log_has( + 'Saving Trials to \'freqtrade/tests/optimize/ut_trials.pickle\'', + caplog.record_tuples + ) + mock_dump.assert_called_once() + + +def test_read_trials_returns_trials_file(mocker, default_conf, caplog) -> None: trials = create_trials(mocker) - mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', - return_value=None) - trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', - return_value='ut_trials.pickle') - mocker.patch('freqtrade.optimize.hyperopt.open', - return_value=trials_path) - save_trials(trials, trials_path) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load) - mock_dump.assert_called_once_with(trials, trials_path) - - -def test_read_trials_returns_trials_file(mocker): - trials = create_trials(mocker) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', - return_value=trials) - mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', - return_value=mock_load) - - assert read_trials() == trials + hyperopt = _HYPEROPT + hyperopt_trial = hyperopt.read_trials() + assert tt.log_has( + 'Reading Trials from \'freqtrade/tests/optimize/ut_trials.pickle\'', + caplog.record_tuples + ) + assert hyperopt_trial == trials mock_open.assert_called_once() mock_load.assert_called_once() -def test_roi_table_generation(): +def test_roi_table_generation() -> None: params = { 'roi_t1': 5, 'roi_t2': 10, @@ -249,4 +279,49 @@ def test_roi_table_generation(): 'roi_p2': 2, 'roi_p3': 3, } - assert generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0} + + hyperopt = _HYPEROPT + assert hyperopt.generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0} + + +def test_start_calls_fmin(mocker, default_conf) -> None: + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': False}) + conf.update({'timerange': None}) + + hyperopt = Hyperopt(conf) + hyperopt.trials = trials + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() + mock_fmin.assert_called_once() + + +def test_start_uses_mongotrials(mocker, default_conf) -> None: + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + mock_mongotrials = mocker.patch( + 'freqtrade.optimize.hyperopt.MongoTrials', + return_value=create_trials(mocker) + ) + + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': True}) + conf.update({'timerange': None}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + + hyperopt = Hyperopt(conf) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() + mock_mongotrials.assert_called_once() + mock_fmin.assert_called_once() diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index b1459198d..4e35507f5 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -6,7 +6,6 @@ import logging import uuid from shutil import copyfile from freqtrade import optimize -from freqtrade.analyze import Analyze from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ download_backtesting_testdata, load_tickerdata_file, trim_tickerlist from freqtrade.misc import file_dump_json @@ -220,16 +219,6 @@ def test_init(default_conf, mocker) -> None: ) -def test_tickerdata_to_dataframe(default_conf) -> None: - analyze = Analyze(default_conf) - - timerange = ((None, 'line'), None, -100) - tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) - tickerlist = {'BTC_UNITEST': tick} - data = analyze.tickerdata_to_dataframe(tickerlist) - assert len(data['BTC_UNITEST']) == 100 - - def test_trim_tickerlist() -> None: with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: ticker_list = json.load(data_file) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 3c1687ab3..7ada49028 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,8 +10,9 @@ import logging import arrow from pandas import DataFrame -import freqtrade.tests.conftest as tt # test tools from freqtrade.analyze import Analyze, SignalType +from freqtrade.optimize.__init__ import load_tickerdata_file +import freqtrade.tests.conftest as tt # test tools # Avoid to reinit the same object again and again @@ -173,3 +174,16 @@ def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv): # Test file without BV data dataframe = Analyze.parse_ticker_dataframe(ticker_history_without_bv) assert dataframe.columns.tolist() == columns + + +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Analyze.tickerdata_to_dataframe() method + """ + analyze = Analyze(default_conf) + + timerange = ((None, 'line'), None, -100) + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) + tickerlist = {'BTC_UNITEST': tick} + data = analyze.tickerdata_to_dataframe(tickerlist) + assert len(data['BTC_UNITEST']) == 100 diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 9af42a30e..fcec2d119 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,17 +1,19 @@ # pragma pylint: disable=missing-docstring, C0103 import pandas -import freqtrade.optimize -from freqtrade import analyze +from freqtrade.optimize import load_data +from freqtrade.analyze import Analyze, SignalType _pairs = ['BTC_ETH'] def load_dataframe_pair(pairs): - ld = freqtrade.optimize.load_data(None, ticker_interval=5, pairs=pairs) + ld = load_data(None, ticker_interval=5, pairs=pairs) assert isinstance(ld, dict) assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] + + analyze = Analyze({'strategy': 'default_strategy'}) dataframe = analyze.analyze_ticker(dataframe) return dataframe From bc8ca491cdb057889d7b69431dd509be14bdd801 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 2 Mar 2018 21:47:00 +0800 Subject: [PATCH 14/56] Minor updates --- freqtrade/freqtradebot.py | 3 ++- freqtrade/tests/conftest.py | 6 ++++-- freqtrade/tests/test_freqtradebot.py | 7 +++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 35acb7649..4449e9b7f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -2,6 +2,7 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() """ +import logging import arrow import copy import json @@ -36,7 +37,7 @@ class FreqtradeBot(object): """ # Init the logger - self.logger = Logger(name='freqtrade').get_logger() + self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() # Init bot states self._state = State.STOPPED diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 722017a15..3ca953156 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -8,6 +8,7 @@ import arrow import pytest from jsonschema import validate from telegram import Chat, Message, Update +from sqlalchemy import create_engine from freqtrade.analyze import Analyze from freqtrade.constants import Constants @@ -41,7 +42,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) - return FreqtradeBot(config) + return FreqtradeBot(config, create_engine('sqlite://')) @pytest.fixture(scope="module") @@ -83,7 +84,8 @@ def default_conf(): "token": "token", "chat_id": "0" }, - "initial_state": "running" + "initial_state": "running", + "loglevel": logging.DEBUG } validate(configuration, Constants.CONF_SCHEMA) return configuration diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5177bd3b7..c1a6819aa 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -8,7 +8,7 @@ import logging import time from unittest.mock import MagicMock from copy import deepcopy -from typing import Optional +from typing import Dict, Optional import arrow import pytest import requests @@ -36,7 +36,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) patch_pymarketcap(mocker) - return FreqtradeBot(config) + return FreqtradeBot(config, create_engine('sqlite://')) def patch_get_signal(mocker, value=(True, False)) -> None: @@ -63,7 +63,7 @@ def patch_RPCManager(mocker) -> MagicMock: return rpc_mock -def patch_pymarketcap(mocker, value: Optional[str] = None) -> None: +def patch_pymarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: """ Mocker to Pymarketcap to speed up tests :param mocker: mocker to patch Pymarketcap class @@ -530,7 +530,6 @@ def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> No """ Test exception on process_maybe_execute_buy() method """ - caplog.set_level(logging.INFO) freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch( From 8bd0f4d0d7999ec937828d1be4166f8b54673df7 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 2 Mar 2018 22:20:26 +0800 Subject: [PATCH 15/56] Remove ugly pprints --- freqtrade/optimize/backtesting.py | 4 ---- freqtrade/rpc/telegram.py | 2 -- freqtrade/tests/optimize/test_backtesting.py | 4 ---- freqtrade/tests/test_configuration.py | 4 ---- 4 files changed, 14 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c15aee1fd..2e7f4ecd1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -253,10 +253,6 @@ class Backtesting(object): # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) - - import pprint - pprint.pprint(min_date) - pprint.pprint(max_date) self.logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c0f5f815b..e721784d1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -109,8 +109,6 @@ class Telegram(RPC): if not self.is_enabled(): return - import pprint - pprint.pprint(self._updater.stop.call_count) self._updater.stop() def is_enabled(self) -> bool: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index d4f705c51..f6968d370 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -186,10 +186,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'refresh_pairs'in config assert tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) - - import pprint - pprint.pprint(caplog.record_tuples) - pprint.pprint(config['timerange']) assert 'timerange' in config assert tt.log_has( 'Parameter --timerange detected: {} ...'.format(config['timerange']), diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 9d5970f25..de014dd6f 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -259,10 +259,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'refresh_pairs'in config assert tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) - - import pprint - pprint.pprint(caplog.record_tuples) - pprint.pprint(config['timerange']) assert 'timerange' in config assert tt.log_has( 'Parameter --timerange detected: {} ...'.format(config['timerange']), From 6148f989800ab2cabefa8c7a64b2d05f15fb0979 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 2 Mar 2018 22:48:18 +0800 Subject: [PATCH 16/56] Fix Telegram unit test when using an internet connection --- freqtrade/tests/rpc/test_rpc_telegram.py | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 5aaf13742..353a44c37 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -143,10 +143,14 @@ def test_is_not_enabled(default_conf, mocker) -> None: assert not telegram.is_enabled() -def test_authorized_only(default_conf, caplog) -> None: +def test_authorized_only(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are authorized """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + chat = Chat(0, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) @@ -170,10 +174,14 @@ def test_authorized_only(default_conf, caplog) -> None: ) -def test_authorized_only_unauthorized(default_conf, caplog) -> None: +def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are unauthorized """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) @@ -197,10 +205,14 @@ def test_authorized_only_unauthorized(default_conf, caplog) -> None: ) -def test_authorized_only_exception(default_conf, caplog) -> None: +def test_authorized_only_exception(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when an exception is thrown """ + patch_get_signal(mocker, (True, False)) + patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), Chat(0, 0)) @@ -585,6 +597,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: patch_get_signal(mocker, (True, False)) patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=mock_balance) mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', side_effect=mock_ticker) @@ -615,6 +628,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: """ patch_get_signal(mocker, (True, False)) patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=[]) msg_mock = MagicMock() @@ -638,6 +652,7 @@ def test_start_handle(default_conf, update, mocker) -> None: Test _start() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -661,6 +676,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: Test _start() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -685,6 +701,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: Test _stop() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -709,6 +726,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: Test _stop() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -993,6 +1011,7 @@ def test_help_handle(default_conf, update, mocker) -> None: Test _help() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -1012,6 +1031,7 @@ def test_version_handle(default_conf, update, mocker) -> None: Test _version() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -1031,6 +1051,7 @@ def test_send_msg(default_conf, mocker) -> None: Test send_msg() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) bot = MagicMock() @@ -1052,6 +1073,7 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None: Test send_msg() method """ patch_pymarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) bot = MagicMock() From d274f134804d9092f7155df33c2331b71deb4d5a Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 2 Mar 2018 22:53:36 +0800 Subject: [PATCH 17/56] Remove Memory profiler in Backtesting --- freqtrade/optimize/backtesting.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2e7f4ecd1..578b7b18d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -19,7 +19,6 @@ from freqtrade.logger import Logger from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade -from memory_profiler import profile class Backtesting(object): """ @@ -221,7 +220,6 @@ class Backtesting(object): labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) - @profile(precision=10) def start(self) -> None: """ Run a backtesting end-to-end From 390501bac02f8e7d8fff1d20d62a3add82386a08 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 2 Mar 2018 23:22:00 +0800 Subject: [PATCH 18/56] Make Pylint Happy chapter 1 --- freqtrade/analyze.py | 7 +- freqtrade/freqtradebot.py | 37 +++---- freqtrade/optimize/__init__.py | 26 +++-- freqtrade/optimize/hyperopt.py | 20 ++-- freqtrade/persistence.py | 7 ++ freqtrade/rpc/rpc.py | 128 ++++++++++++---------- freqtrade/rpc/rpc_manager.py | 1 - freqtrade/rpc/telegram.py | 63 ++++++----- freqtrade/strategy/strategy.py | 2 +- freqtrade/tests/optimize/test_hyperopt.py | 10 +- freqtrade/tests/test_configuration.py | 2 +- freqtrade/tests/test_constants.py | 3 - freqtrade/tests/test_dataframe.py | 2 +- 13 files changed, 161 insertions(+), 147 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index bde0c9f80..bed644e03 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -1,11 +1,11 @@ """ Functions to analyze ticker data with indicators and produce buy and sell signals """ -import arrow from datetime import datetime, timedelta from enum import Enum -from pandas import DataFrame, to_datetime from typing import Dict, List +import arrow +from pandas import DataFrame, to_datetime from freqtrade.exchange import get_ticker_history from freqtrade.logger import Logger from freqtrade.strategy.strategy import Strategy @@ -188,10 +188,9 @@ class Analyze(object): ) return False - def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: """ Creates a dataframe and populates indicators for given ticker data """ return {pair: self.populate_indicators(self.parse_ticker_dataframe(pair_data)) - for pair, pair_data in tickerdata.items()} \ No newline at end of file + for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4449e9b7f..4b3201514 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -2,16 +2,15 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() """ -import logging -import arrow import copy import json -import requests import time import traceback -from cachetools import cached, TTLCache -from datetime import datetime from typing import Dict, List, Optional, Any, Callable +from datetime import datetime +import requests +import arrow +from cachetools import cached, TTLCache from freqtrade.analyze import Analyze from freqtrade.constants import Constants from freqtrade.fiat_convert import CryptoToFiatConverter @@ -507,14 +506,14 @@ class FreqtradeBot(object): "*Current Rate:* `{current_rate:.8f}`\n" \ "*Profit:* `{profit:.2f}%`" \ "".format( - exchange=trade.exchange, - pair=trade.pair, - pair_url=exchange.get_pair_detail_url(trade.pair), - limit=limit, - open_rate=trade.open_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - profit=round(profit * 100, 2), + exchange=trade.exchange, + pair=trade.pair, + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit, + open_rate=trade.open_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + profit=round(profit * 100, 2), ) # For regular case, when the configuration exists @@ -528,12 +527,12 @@ class FreqtradeBot(object): message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ '` / {profit_fiat:.3f} {fiat})`' \ ''.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade, - coin=self.config['stake_currency'], - profit_fiat=profit_fiat, - fiat=self.config['fiat_display_currency'], + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade, + coin=self.config['stake_currency'], + profit_fiat=profit_fiat, + fiat=self.config['fiat_display_currency'], ) # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e7312292e..7d3c18a06 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -97,10 +97,11 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: try: download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval) except BaseException: - logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format( - pair=pair, - interval=ticker_interval, - )) + logger.info( + 'Failed to download the pair: "%s", Interval: %s min', + pair, + ticker_interval + ) return False return True @@ -115,10 +116,11 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> """ path = make_testdata_path(datadir) - logger.info('Download the pair: "{pair}", Interval: {interval} min'.format( - pair=pair, - interval=interval, - )) + logger.info( + 'Download the pair: "%s", Interval: %s min', + pair, + interval + ) filepair = pair.replace("-", "_") filename = os.path.join(path, '{pair}-{interval}.json'.format( @@ -129,8 +131,8 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> if os.path.isfile(filename): with open(filename, "rt") as file: data = json.load(file) - logger.debug("Current Start: {}".format(data[1]['T'])) - logger.debug("Current End: {}".format(data[-1:][0]['T'])) + logger.debug("Current Start: %s", data[1]['T']) + logger.debug("Current End: %s", data[-1:][0]['T']) else: data = [] logger.debug("Current Start: None") @@ -140,8 +142,8 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> for row in new_data: if row not in data: data.append(row) - logger.debug("New Start: {}".format(data[1]['T'])) - logger.debug("New End: {}".format(data[-1:][0]['T'])) + logger.debug("New Start: %s", data[1]['T']) + logger.debug("New End: %s", data[-1:][0]['T']) data = sorted(data, key=lambda data: data['T']) misc.file_dump_json(filename, data) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c1d95d6ef..cf68c19f8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -25,7 +25,7 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.arguments import Arguments -from freqtrade.optimize.backtesting import Backtesting, setup_configuration +from freqtrade.optimize.backtesting import Backtesting from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf @@ -46,7 +46,6 @@ class Hyperopt(Backtesting): self.logging = Logger(name=__name__, level=config['loglevel']) self.logger = self.logging.get_logger() - # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 @@ -353,6 +352,9 @@ class Hyperopt(Backtesting): Define the buy strategy parameters to be used by hyperopt """ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ conditions = [] # GUARDS AND TRENDS if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: @@ -513,8 +515,9 @@ class Hyperopt(Backtesting): self.current_tries = len(self.trials.results) self.total_tries += self.current_tries self.logger.info( - 'Continuing with trials. Current: {}, Total: {}' - .format(self.current_tries, self.total_tries) + 'Continuing with trials. Current: %d, Total: %d', + self.current_tries, + self.total_tries ) try: @@ -557,7 +560,10 @@ class Hyperopt(Backtesting): """ Hyperopt SIGINT handler """ - self.logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + self.logger.info( + 'Hyperopt received %s', + signal.Signals(sig).name + ) self.save_trials() self.log_trials_result() @@ -580,9 +586,7 @@ def start(args) -> None: logger.info('Starting freqtrade in Hyperopt mode') # Initialize configuration - #config = setup_configuration(args) - - # Monkey patch of the configuration with hyperopt_conf.py + # Monkey patch the configuration with hyperopt_conf.py configuration = Configuration(args) optimize_config = hyperopt_optimize_conf() config = configuration._load_backtesting_config(optimize_config) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 79903d66f..ea6603795 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -1,3 +1,7 @@ +""" +This module contains the class to persist trades into SQLite +""" + import logging from datetime import datetime from decimal import Decimal, getcontext @@ -72,6 +76,9 @@ def clean_dry_run_db() -> None: class Trade(_DECL_BASE): + """ + Class used to define a trade structure + """ __tablename__ = 'trades' id = Column(Integer, primary_key=True) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 30acf0bf4..fa9c2022d 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -2,9 +2,9 @@ This module contains class to define a RPC communications """ -import arrow from decimal import Decimal from datetime import datetime, timedelta +import arrow from pandas import DataFrame import sqlalchemy as sql from freqtrade.logger import Logger @@ -18,7 +18,6 @@ class RPC(object): """ RPC class can be used to have extra feature, like bot data, and access to DB data """ - def __init__(self, freqtrade) -> None: """ Initializes all enabled rpc modules @@ -65,21 +64,21 @@ class RPC(object): "*Close Profit:* `{close_profit}`\n" \ "*Current Profit:* `{current_profit:.2f}%`\n" \ "*Open Order:* `{open_order}`"\ - .format( - trade_id=trade.id, - pair=trade.pair, - market_url=exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit=fmt_close_profit, - current_profit=round(current_profit * 100, 2), - open_order='({} rem={:.8f})'.format( - order['type'], order['remaining'] - ) if order else None, - ) + .format( + trade_id=trade.id, + pair=trade.pair, + market_url=exchange.get_pair_detail_url(trade.pair), + date=arrow.get(trade.open_date).humanize(), + open_rate=trade.open_rate, + close_rate=trade.close_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + close_profit=fmt_close_profit, + current_profit=round(current_profit * 100, 2), + open_order='({} rem={:.8f})'.format( + order['type'], order['remaining'] + ) if order else None, + ) result.append(message) return (False, result) @@ -100,7 +99,7 @@ class RPC(object): shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) ]) - + columns = ['ID', 'Pair', 'Since', 'Profit'] df_statuses = DataFrame.from_records(trades_list, columns=columns) df_statuses = df_statuses.set_index(columns[0]) @@ -113,10 +112,10 @@ class RPC(object): def rpc_daily_profit(self, timescale, stake_currency, fiat_display_currency): today = datetime.utcnow().date() profit_days = {} - + if not (isinstance(timescale, int) and timescale > 0): return (True, '*Daily [n]:* `must be an integer greater than 0`') - + fiat = self.freqtrade.fiat_converter for day in range(0, timescale): profitday = today - timedelta(days=day) @@ -131,7 +130,7 @@ class RPC(object): 'amount': format(curdayprofit, '.8f'), 'trades': len(trades) } - + stats = [ [ key, @@ -147,7 +146,10 @@ class RPC(object): ), symbol=fiat_display_currency ), - '{value} trade{s}'.format(value=value['trades'], s='' if value['trades'] < 2 else 's'), + '{value} trade{s}'.format( + value=value['trades'], + s='' if value['trades'] < 2 else 's' + ), ] for key, value in profit_days.items() ] @@ -158,21 +160,21 @@ class RPC(object): :return: cumulative profit statistics. """ trades = Trade.query.order_by(Trade.id).all() - + profit_all_coin = [] profit_all_percent = [] profit_closed_coin = [] profit_closed_percent = [] durations = [] - + for trade in trades: current_rate = None - + if not trade.open_rate: continue if trade.close_date: durations.append((trade.close_date - trade.open_date).total_seconds()) - + if not trade.is_open: profit_percent = trade.calc_profit_percent() profit_closed_coin.append(trade.calc_profit()) @@ -181,22 +183,25 @@ class RPC(object): # Get current rate current_rate = exchange.get_ticker(trade.pair, False)['bid'] profit_percent = trade.calc_profit_percent(rate=current_rate) - - profit_all_coin.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate))) + + profit_all_coin.append( + trade.calc_profit(rate=Decimal(trade.close_rate or current_rate)) + ) profit_all_percent.append(profit_percent) - - best_pair = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .first() - + + best_pair = Trade.session.query( + Trade.pair, + sql.func.sum(Trade.close_profit).label('profit_sum') + )\ + .filter(Trade.is_open.is_(False))\ + .group_by(Trade.pair)\ + .order_by(sql.text('profit_sum DESC')).first() + if not best_pair: return (True, '*Status:* `no closed trade`') - + bp_pair, bp_rate = best_pair - + # FIX: we want to keep fiatconverter in a state/environment, # doing this will utilize its caching functionallity, instead we reinitialize it here fiat = self.freqtrade.fiat_converter @@ -244,7 +249,7 @@ class RPC(object): ] if not balances: return (True, '`All balances are zero.`') - + output = [] total = 0.0 for currency in balances: @@ -258,12 +263,15 @@ class RPC(object): currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid'] currency['BTC'] = currency["Rate"] * currency["Balance"] total = total + currency['BTC'] - output.append({'currency': currency['Currency'], - 'available': currency['Available'], - 'balance': currency['Balance'], - 'pending': currency['Pending'], - 'est_btc': currency['BTC'] - }) + output.append( + { + 'currency': currency['Currency'], + 'available': currency['Available'], + 'balance': currency['Balance'], + 'pending': currency['Pending'], + 'est_btc': currency['BTC'] + } + ) fiat = self.freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) @@ -275,9 +283,9 @@ class RPC(object): """ if self.freqtrade.get_state() == State.RUNNING: return (True, '*Status:* `already running`') - else: - self.freqtrade.update_state(State.RUNNING) - return (False, '`Starting trader ...`') + + self.freqtrade.update_state(State.RUNNING) + return (False, '`Starting trader ...`') def rpc_stop(self) -> (bool, str): """ @@ -286,8 +294,8 @@ class RPC(object): if self.freqtrade.get_state() == State.RUNNING: self.freqtrade.update_state(State.STOPPED) return (False, '`Stopping trader ...`') - else: - return (True, '*Status:* `already stopped`') + + return (True, '*Status:* `already stopped`') # FIX: no test for this!!!! def rpc_forcesell(self, trade_id) -> None: @@ -300,18 +308,18 @@ class RPC(object): # Check if there is there is an open order if trade.open_order_id: order = exchange.get_order(trade.open_order_id) - + # Cancel open LIMIT_BUY orders and close trade if order and not order['closed'] and order['type'] == 'LIMIT_BUY': exchange.cancel_order(trade.open_order_id) trade.close(order.get('rate') or trade.open_rate) # TODO: sell amount which has been bought already return - + # Ignore trades with an attached LIMIT_SELL order if order and not order['closed'] and order['type'] == 'LIMIT_SELL': return - + # Get current rate and execute sell current_rate = exchange.get_ticker(trade.pair, False)['bid'] self.freqtrade.execute_sell(trade, current_rate) @@ -319,13 +327,13 @@ class RPC(object): if self.freqtrade.get_state() != State.RUNNING: return (True, '`trader is not running`') - + if trade_id == 'all': # Execute sell for all open orders for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): _exec_forcesell(trade) return (False, '') - + # Query for trade trade = Trade.query.filter( sql.and_( @@ -336,7 +344,7 @@ class RPC(object): if not trade: self.logger.warning('forcesell: Invalid argument received') return (True, 'Invalid argument.') - + _exec_forcesell(trade) return (False, '') @@ -347,7 +355,7 @@ class RPC(object): """ if self.freqtrade.get_state() != State.RUNNING: return (True, '`trader is not running`') - + pair_rates = Trade.session.query(Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum'), sql.func.count(Trade.pair).label('count')) \ @@ -358,9 +366,9 @@ class RPC(object): trades = [] for (pair, rate, count) in pair_rates: trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) - + return (False, trades) - + def rpc_count(self) -> None: """ Returns the number of trades running @@ -368,6 +376,6 @@ class RPC(object): """ if self.freqtrade.get_state() != State.RUNNING: return (True, '`trader is not running`') - + trades = Trade.query.filter(Trade.is_open.is_(True)).all() return (False, trades) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 5987d8c4d..cd3a694c2 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -10,7 +10,6 @@ class RPCManager(object): """ Class to manage RPC objects (Telegram, Slack, ...) """ - def __init__(self, freqtrade) -> None: """ Initializes all enabled rpc modules diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e721784d1..fa1d290b9 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,14 +1,16 @@ +# pragma pylint: disable=unused-argument, unused-variable, protected-access, invalid-name + """ This module manage Telegram communication """ from typing import Any, Callable -from freqtrade.rpc.rpc import RPC from tabulate import tabulate from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ +from freqtrade.rpc.rpc import RPC def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: @@ -17,10 +19,10 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ :param command_handler: Telegram CommandHandler :return: decorated function """ - - #def wrapper(self, bot: Bot, update: Update): def wrapper(self, *args, **kwargs): - + """ + Decorator logic + """ update = kwargs.get('update') or args[1] # Reject unauthorized messages @@ -45,6 +47,7 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ return wrapper + class Telegram(RPC): """ Telegram, this class send messages to Telegram @@ -57,7 +60,7 @@ class Telegram(RPC): """ super().__init__(freqtrade) - self._updater = Updater = None + self._updater = None self._config = freqtrade.config self._init() @@ -190,10 +193,10 @@ class Telegram(RPC): ], tablefmt='simple') message = 'Daily Profit over the last {} days:\n
{}
'\ - .format( - timescale, - stats - ) + .format( + timescale, + stats + ) self.send_msg(message, bot=bot, parse_mode=ParseMode.HTML) @authorized_only @@ -225,22 +228,22 @@ class Telegram(RPC): "*Latest Trade opened:* `{latest_trade_date}`\n" \ "*Avg. Duration:* `{avg_duration}`\n" \ "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ - .format( - coin=self._config['stake_currency'], - fiat=self._config['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) + .format( + coin=self._config['stake_currency'], + fiat=self._config['fiat_display_currency'], + profit_closed_coin=stats['profit_closed_coin'], + profit_closed_percent=stats['profit_closed_percent'], + profit_closed_fiat=stats['profit_closed_fiat'], + profit_all_coin=stats['profit_all_coin'], + profit_all_percent=stats['profit_all_percent'], + profit_all_fiat=stats['profit_all_fiat'], + trade_count=stats['trade_count'], + first_trade_date=stats['first_trade_date'], + latest_trade_date=stats['latest_trade_date'], + avg_duration=stats['avg_duration'], + best_pair=stats['best_pair'], + best_rate=stats['best_rate'] + ) self.send_msg(markdown_msg, bot=bot) @authorized_only @@ -294,7 +297,6 @@ class Telegram(RPC): (error, msg) = self.rpc_stop() self.send_msg(msg, bot=bot) - # FIX: no test for this!!!! @authorized_only def _forcesell(self, bot: Bot, update: Update) -> None: """ @@ -370,10 +372,12 @@ class Telegram(RPC): "*/status [table]:* `Lists all open trades`\n" \ " *table :* `will display trades in a table`\n" \ "*/profit:* `Lists cumulative profit from all finished trades`\n" \ - "*/forcesell |all:* `Instantly sells the given trade or all trades, regardless of profit`\n" \ + "*/forcesell |all:* `Instantly sells the given trade or all trades, " \ + "regardless of profit`\n" \ "*/performance:* `Show performance of each finished trade grouped by pair`\n" \ "*/daily :* `Shows profit or loss per day, over the last n days`\n" \ - "*/count:* `Show number of trades running compared to allowed number of trades`\n" \ + "*/count:* `Show number of trades running compared to allowed number of trades`" \ + "\n" \ "*/balance:* `Show account balance per currency`\n" \ "*/help:* `This help message`\n" \ "*/version:* `Show version`" @@ -391,7 +395,8 @@ class Telegram(RPC): """ self.send_msg('*Version:* `{}`'.format(__version__), bot=bot) - def send_msg(self, msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + def send_msg(self, msg: str, bot: Bot = None, + parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: """ Send given markdown message :param msg: message diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 8333beea7..24ad31190 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -19,7 +19,7 @@ class Strategy(object): """ This class contains all the logic to load custom strategy class """ - def __init__(self, config: dict={}) -> None: + def __init__(self, config: dict = {}) -> None: """ Load the custom class from config parameter :param config: diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 63e4404ff..30b5e4382 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,14 +1,8 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import logging import os -import pytest from copy import deepcopy - -#from freqtrade.optimize.hyperopt import EXPECTED_MAX_PROFIT, start, \ -# log_results, save_trials, read_trials, generate_roi_table from unittest.mock import MagicMock - -from freqtrade.optimize.hyperopt import Hyperopt, start +from freqtrade.optimize.hyperopt import Hyperopt import freqtrade.tests.conftest as tt # test tools @@ -24,7 +18,7 @@ def create_trials(mocker) -> None: - we might have a pickle'd file so make sure that we return false when looking for it """ - _HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize','ut_trials.pickle') + _HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index de014dd6f..e6ca8bfd0 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -1,4 +1,5 @@ # pragma pylint: disable=protected-access, invalid-name + """ Unit test file for configuration.py """ @@ -270,4 +271,3 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'Parameter --export detected: {} ...'.format(config['export']), caplog.record_tuples ) - diff --git a/freqtrade/tests/test_constants.py b/freqtrade/tests/test_constants.py index 4c94926bc..6d544502f 100644 --- a/freqtrade/tests/test_constants.py +++ b/freqtrade/tests/test_constants.py @@ -8,7 +8,6 @@ from freqtrade.constants import Constants def test_constant_object() -> None: """ Test the Constants object has the mandatory Constants - :return: None """ assert hasattr(Constants, 'CONF_SCHEMA') assert hasattr(Constants, 'DYNAMIC_WHITELIST') @@ -19,11 +18,9 @@ def test_constant_object() -> None: assert hasattr(Constants, 'DEFAULT_STRATEGY') - def test_conf_schema() -> None: """ Test the CONF_SCHEMA is from the right type - :return: """ constant = Constants() assert isinstance(constant.CONF_SCHEMA, dict) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index fcec2d119..eaa761bdb 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -2,7 +2,7 @@ import pandas from freqtrade.optimize import load_data -from freqtrade.analyze import Analyze, SignalType +from freqtrade.analyze import Analyze _pairs = ['BTC_ETH'] From 5b314e2f7a9f0877527a04e66b4767ca0cf5e92c Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Mar 2018 09:18:34 +0800 Subject: [PATCH 19/56] Port commit "Remove Strategy fallback to default strategy (#490)" Hash: d24cd893048947503d330643e1628b0b5c5223eb --- docs/bot-usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 204b35c3a..cf3258465 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -65,8 +65,8 @@ load it: python3 ./freqtrade/main.py --strategy my_awesome_strategy ``` -If the bot does not find your strategy file, it will display in an error - message the reason (File not found, or errors in your code). +If the bot does not find your strategy file, it will display in an error +message the reason (File not found, or errors in your code). Learn more about strategy file in [optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). From 84759073d9a65ffd3ce6acb99e30c62afa4593c0 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 3 Mar 2018 13:39:39 -0800 Subject: [PATCH 20/56] Refactor Configuration() to apply common configurations all the time and to remove show_info --- freqtrade/configuration.py | 78 +++++++++++++++------------ freqtrade/misc.py | 1 - freqtrade/optimize/backtesting.py | 1 + freqtrade/optimize/hyperopt.py | 3 +- freqtrade/tests/test_configuration.py | 48 +++++++---------- freqtrade/tests/test_main.py | 10 ++-- 6 files changed, 71 insertions(+), 70 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 721def2a4..dfce90bf6 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -17,14 +17,13 @@ class Configuration(object): Class to read and init the bot configuration Reuse this class for the bot, backtesting, hyperopt and every script that required configuration """ - def __init__(self, args: List[str]) -> None: + def __init__(self, args: List[str], do_not_init=False) -> None: self.args = args self.logging = Logger(name=__name__) self.logger = self.logging.get_logger() - self.config = self._load_config() - self.show_info() + self.config = None - def _load_config(self) -> Dict[str, Any]: + def load_config(self) -> Dict[str, Any]: """ Extract information for sys.argv and load the bot configuration :return: Configuration dictionary @@ -35,18 +34,8 @@ class Configuration(object): # Add the strategy file to use config.update({'strategy': self.args.strategy}) - # Add dynamic_whitelist if found - if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: - config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) - - # Add dry_run_db if found and the bot in dry run - if self.args.dry_run_db and config.get('dry_run', False): - config.update({'dry_run_db': True}) - - # Log level - if 'loglevel' in self.args and self.args.loglevel: - config.update({'loglevel': self.args.loglevel}) - self.logging.set_level(self.args.loglevel) + # Load Common configuration + config = self._load_common_config(config) # Load Backtesting config = self._load_backtesting_config(config) @@ -71,11 +60,46 @@ class Configuration(object): return self._validate_config(conf) + def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load common configuration + :return: configuration as dictionary + """ + + # Log level + if 'loglevel' in self.args and self.args.loglevel: + config.update({'loglevel': self.args.loglevel}) + self.logging.set_level(self.args.loglevel) + self.logger.info('Log level set at %s', config['loglevel']) + + # Add dynamic_whitelist if found + if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: + config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) + self.logger.info( + 'Parameter --dynamic-whitelist detected. ' + 'Using dynamically generated whitelist. ' + '(not applicable with Backtesting and Hyperopt)' + ) + + # Add dry_run_db if found and the bot in dry run + if self.args.dry_run_db and config.get('dry_run', False): + config.update({'dry_run_db': True}) + self.logger.info('Parameter --dry-run-db detected ...') + + if config.get('dry_run_db', False): + if config.get('dry_run', False): + self.logger.info('Dry_run will use the DB file: "tradesv3.dry_run.sqlite"') + else: + self.logger.info('Dry run is disabled. (--dry_run_db ignored)') + + return config + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Backtesting configuration :return: configuration as dictionary """ + # If -i/--ticker-interval is used we override the configuration parameter # (that will override the strategy configuration) if 'ticker_interval' in self.args and self.args.ticker_interval: @@ -152,28 +176,12 @@ class Configuration(object): best_match(Draft4Validator(Constants.CONF_SCHEMA).iter_errors(conf)).message ) - def show_info(self) -> None: - """ - Display info message to user depending of the configuration of the bot - :return: None - """ - if self.config.get('dynamic_whitelist', False): - self.logger.info( - 'Using dynamically generated whitelist. (--dynamic-whitelist detected)' - ) - - if self.config.get('dry_run_db', False): - if self.config.get('dry_run', False): - self.logger.info( - 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". ' - '(--dry_run_db detected)' - ) - else: - self.logger.info('Dry run is disabled. (--dry_run_db ignored)') - def get_config(self) -> Dict[str, Any]: """ Return the config. Use this method to get the bot config :return: Dict: Bot config """ + if self.config is None: + self.config = self.load_config() + return self.config diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 55bfed1f7..741a306a1 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -69,4 +69,3 @@ def file_dump_json(filename, data) -> None: """ with open(filename, 'w') as file: json.dump(data, file) - diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index dfe5a5f49..51cc7c053 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -293,6 +293,7 @@ def setup_configuration(args) -> Dict[str, Any]: :return: Configuration """ configuration = Configuration(args) + config = configuration.get_config() # Ensure we do not use Exchange credentials diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3408a63c3..ad8ff32be 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -589,7 +589,8 @@ def start(args) -> None: # Monkey patch the configuration with hyperopt_conf.py configuration = Configuration(args) optimize_config = hyperopt_optimize_conf() - config = configuration._load_backtesting_config(optimize_config) + config = configuration._load_common_config(optimize_config) + config = configuration._load_backtesting_config(config) config = configuration._load_hyperopt_config(config) config['exchange']['key'] = '' config['exchange']['secret'] = '' diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e6ca8bfd0..cb50ea72d 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,7 +6,6 @@ Unit test file for configuration.py import json from copy import deepcopy -from unittest.mock import MagicMock import pytest from jsonschema import ValidationError @@ -19,10 +18,12 @@ def test_configuration_object() -> None: """ Test the Constants object has the mandatory Constants """ - assert hasattr(Configuration, '_load_config') + assert hasattr(Configuration, 'load_config') assert hasattr(Configuration, '_load_config_file') assert hasattr(Configuration, '_validate_config') - assert hasattr(Configuration, 'show_info') + assert hasattr(Configuration, '_load_common_config') + assert hasattr(Configuration, '_load_backtesting_config') + assert hasattr(Configuration, '_load_hyperopt_config') assert hasattr(Configuration, 'get_config') @@ -30,11 +31,6 @@ def test_load_config_invalid_pair(default_conf, mocker) -> None: """ Test the configuration validator with an invalid PAIR format """ - mocker.patch.multiple( - 'freqtrade.configuration.Configuration', - _load_config=MagicMock(return_value=[]), - show_info=MagicMock - ) conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'].append('BTC-ETH') @@ -47,11 +43,6 @@ def test_load_config_missing_attributes(default_conf, mocker) -> None: """ Test the configuration validator with a missing attribute """ - mocker.patch.multiple( - 'freqtrade.configuration.Configuration', - _load_config=MagicMock(return_value=[]), - show_info=MagicMock - ) conf = deepcopy(default_conf) conf.pop('exchange') @@ -64,11 +55,6 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: """ Test Configuration._load_config_file() method """ - mocker.patch.multiple( - 'freqtrade.configuration.Configuration', - _load_config=MagicMock(return_value=[]), - show_info=MagicMock - ) file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -83,7 +69,7 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: def test_load_config(default_conf, mocker) -> None: """ - Test Configuration._load_config() without any cli params + Test Configuration.load_config() without any cli params """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) @@ -91,7 +77,7 @@ def test_load_config(default_conf, mocker) -> None: args = Arguments([], '').get_parsed_arg() configuration = Configuration(args) - validated_conf = configuration._load_config() + validated_conf = configuration.load_config() assert 'strategy' in validated_conf assert validated_conf['strategy'] == 'default_strategy' @@ -101,7 +87,7 @@ def test_load_config(default_conf, mocker) -> None: def test_load_config_with_params(default_conf, mocker) -> None: """ - Test Configuration._load_config() with cli params used + Test Configuration.load_config() with cli params used """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) @@ -115,7 +101,7 @@ def test_load_config_with_params(default_conf, mocker) -> None: args = Arguments(args, '').get_parsed_arg() configuration = Configuration(args) - validated_conf = configuration._load_config() + validated_conf = configuration.load_config() assert 'dynamic_whitelist' in validated_conf assert validated_conf['dynamic_whitelist'] == 10 @@ -141,22 +127,28 @@ def test_show_info(default_conf, mocker, caplog) -> None: args = Arguments(args, '').get_parsed_arg() configuration = Configuration(args) - configuration.show_info() + configuration.get_config() assert tt.log_has( - 'Using dynamically generated whitelist. (--dynamic-whitelist detected)', + 'Parameter --dynamic-whitelist detected. ' + 'Using dynamically generated whitelist. ' + '(not applicable with Backtesting and Hyperopt)', caplog.record_tuples ) assert tt.log_has( - 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". ' - '(--dry_run_db detected)', + 'Parameter --dry-run-db detected ...', + caplog.record_tuples + ) + + assert tt.log_has( + 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite"', caplog.record_tuples ) # Test the Dry run condition configuration.config.update({'dry_run': False}) - configuration.show_info() + configuration._load_common_config(configuration.config) assert tt.log_has( 'Dry run is disabled. (--dry_run_db ignored)', caplog.record_tuples @@ -167,7 +159,6 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> """ Test setup_configuration() function """ - mocker.patch('freqtrade.configuration.Configuration.show_info', MagicMock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -212,7 +203,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non """ Test setup_configuration() function """ - mocker.patch('freqtrade.configuration.Configuration.show_info', MagicMock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index ae89912e3..ef6aabedd 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -29,7 +29,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: """ - Test that main() can start hyperopt. + Test that main() can start hyperopt """ hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock()) main(['hyperopt']) @@ -61,7 +61,7 @@ def test_set_loggers() -> None: def test_main(mocker, caplog) -> None: """ - Test main() function. + Test main() function In this test we are skipping the while True loop by throwing an exception. """ mocker.patch.multiple( @@ -73,9 +73,11 @@ def test_main(mocker, caplog) -> None: clean=MagicMock(), ) + args = ['-c', 'config.json.example'] + # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit) as pytest_wrapped_e: - main([]) + main(args) tt.log_has('Starting freqtrade', caplog.record_tuples) tt.log_has('Got SIGINT, aborting ...', caplog.record_tuples) assert pytest_wrapped_e.type == SystemExit @@ -87,5 +89,5 @@ def test_main(mocker, caplog) -> None: MagicMock(side_effect=BaseException) ) with pytest.raises(SystemExit): - main([]) + main(args) tt.log_has('Got fatal exception!', caplog.record_tuples) From c52e688979331096828fc3c24868c00c9645a51e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 00:58:20 -0800 Subject: [PATCH 21/56] Fix unit tests in test_arguments.py and test_configuration.py --- freqtrade/tests/test_arguments.py | 2 +- freqtrade/tests/test_configuration.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index d3272d0af..940b3811b 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -129,5 +129,5 @@ def test_parse_args_hyperopt_custom() -> None: assert call_args.epochs == 20 assert call_args.loglevel == logging.INFO assert call_args.subparser == 'hyperopt' - assert call_args.spaces == 'buy' + assert call_args.spaces == ['buy'] assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d7420337f..20b377ddf 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -281,5 +281,5 @@ def test_hyperopt_space_argument(mocker, default_conf, caplog) -> None: configuration = Configuration(args) config = configuration.get_config() assert 'spaces' in config - assert config['spaces'] is 'all' - assert tt.log_has('Parameter -s/--spaces detected: all', caplog.record_tuples) + assert config['spaces'] == ['all'] + assert tt.log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) From de468c6fc8f0b7c7efedb72a34c4958b9923f12e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 02:25:40 -0800 Subject: [PATCH 22/56] Fix wrong realistic_simulation implementation in Hyperopt --- config.json.bak | 44 +++++++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 1 - freqtrade/optimize/hyperopt.py | 2 +- 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 config.json.bak diff --git a/config.json.bak b/config.json.bak new file mode 100644 index 000000000..053e83e3f --- /dev/null +++ b/config.json.bak @@ -0,0 +1,44 @@ +{ + "max_open_trades": 3, + "stake_currency": "BTC", + "stake_amount": 0.005, + "fiat_display_currency": "USD", + "dry_run": true, + "unfilledtimeout": 600, + "bid_strategy": { + "ask_last_balance": 0.0 + }, + "exchange": { + "name": "bittrex", + "key": "", + "secret": "", + "pair_whitelist": [ + "BTC_ETH", + "BTC_LTC", + "BTC_ETC", + "BTC_DASH", + "BTC_ZEC", + "BTC_XLM", + "BTC_NXT", + "BTC_POWR", + "BTC_ADA", + "BTC_XMR" + ], + "pair_blacklist": [ + "BTC_DOGE" + ] + }, + "experimental": { + "use_sell_signal": false, + "sell_profit_only": false + }, + "telegram": { + "enabled": true, + "token": "387056091:AAEVz29u5KwphICqGB6c63RwZjqCd7Kh6T4", + "chat_id": "391939601" + }, + "initial_state": "running", + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5ac66c644..92591f98d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -274,7 +274,6 @@ def setup_configuration(args) -> Dict[str, Any]: :return: Configuration """ configuration = Configuration(args) - config = configuration.get_config() # Ensure we do not use Exchange credentials diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f6089a68c..0d9289929 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -447,7 +447,7 @@ class Hyperopt(Backtesting): { 'stake_amount': self.config['stake_amount'], 'processed': self.processed, - 'realistic': params['realistic_simulation'], + 'realistic': self.config.get('realistic_simulation', False), } ) result_explanation = self.format_results(results) From d685646446d4fb2359150dea070b5cbb5301b788 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 17:51:57 -0800 Subject: [PATCH 23/56] Arguments(): Change private methods to public --- freqtrade/arguments.py | 26 +++++++++++++++----------- freqtrade/tests/test_arguments.py | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 8c4624770..30f94fddc 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -21,7 +21,9 @@ class Arguments(object): self.args = args self.parsed_arg = None self.parser = argparse.ArgumentParser(description=description) - self._common_args_parser() + + def _load_args(self): + self.common_args_parser() self._build_subcommands() def get_parsed_arg(self) -> List[str]: @@ -29,11 +31,13 @@ class Arguments(object): Return the list of arguments :return: List[str] List of arguments """ - self.parsed_arg = self._parse_args() + if self.parsed_arg is None: + self._load_args() + self.parsed_arg = self.parse_args() return self.parsed_arg - def _parse_args(self) -> List[str]: + def parse_args(self) -> List[str]: """ Parses given arguments and returns an argparse Namespace instance. """ @@ -41,7 +45,7 @@ class Arguments(object): return parsed_arg - def _common_args_parser(self) -> None: + def common_args_parser(self) -> None: """ Parses given common arguments and returns them as a parsed object. """ @@ -101,7 +105,7 @@ class Arguments(object): ) @staticmethod - def _backtesting_options(parser: argparse.ArgumentParser) -> None: + def backtesting_options(parser: argparse.ArgumentParser) -> None: """ Parses given arguments for Backtesting scripts. """ @@ -128,7 +132,7 @@ class Arguments(object): ) @staticmethod - def _optimizer_shared_options(parser: argparse.ArgumentParser) -> None: + def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: parser.add_argument( '-i', '--ticker-interval', help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', @@ -151,7 +155,7 @@ class Arguments(object): ) @staticmethod - def _hyperopt_options(parser: argparse.ArgumentParser) -> None: + def hyperopt_options(parser: argparse.ArgumentParser) -> None: """ Parses given arguments for Hyperopt scripts. """ @@ -191,14 +195,14 @@ class Arguments(object): # Add backtesting subcommand backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') backtesting_cmd.set_defaults(func=backtesting.start) - self._optimizer_shared_options(backtesting_cmd) - self._backtesting_options(backtesting_cmd) + self.optimizer_shared_options(backtesting_cmd) + self.backtesting_options(backtesting_cmd) # Add hyperopt subcommand hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') hyperopt_cmd.set_defaults(func=hyperopt.start) - self._optimizer_shared_options(hyperopt_cmd) - self._hyperopt_options(hyperopt_cmd) + self.optimizer_shared_options(hyperopt_cmd) + self.hyperopt_options(hyperopt_cmd) @staticmethod def parse_timerange(text: str) -> (List, int, int): diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 940b3811b..6ad507e85 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -17,7 +17,7 @@ def test_arguments_object() -> None: :return: None """ assert hasattr(Arguments, 'get_parsed_arg') - assert hasattr(Arguments, '_parse_args') + assert hasattr(Arguments, 'parse_args') assert hasattr(Arguments, 'parse_timerange') assert hasattr(Arguments, 'scripts_options') From 9ae2491b1e2676f27b4701fe17f858fd7f2a6e6c Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 17:52:58 -0800 Subject: [PATCH 24/56] Plot_dataframe.py: make it works with the new object model --- scripts/plot_dataframe.py | 134 +++++++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 47 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e62465bf7..e7a52b2bf 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -1,26 +1,33 @@ #!/usr/bin/env python3 +""" +Script to display when the bot will buy a specific pair + +Mandatory Cli parameters: +-p / --pair: pair to examine + +Optional Cli parameters +-s / --strategy: strategy to use +-d / --datadir: path to pair backtest data +--timerange: specify what timerange of data to use. +-l / --live: Live, to download the latest ticker for the pair +""" import sys -import logging + +from typing import Dict from plotly import tools from plotly.offline import plot import plotly.graph_objs as go -from freqtrade import exchange, analyze -from freqtrade.strategy.strategy import Strategy -import freqtrade.misc as misc +from freqtrade.arguments import Arguments +from freqtrade.analyze import Analyze +from freqtrade import exchange +from freqtrade.logger import Logger import freqtrade.optimize as optimize -logger = logging.getLogger(__name__) - - -def plot_parse_args(args): - parser = misc.common_args_parser('Graph dataframe') - misc.backtesting_options(parser) - misc.scripts_options(parser) - return parser.parse_args(args) +logger = Logger(name="Graph dataframe").get_logger() def plot_analyzed_dataframe(args) -> None: @@ -30,12 +37,19 @@ def plot_analyzed_dataframe(args) -> None: :return: None """ pair = args.pair.replace('-', '_') - timerange = misc.parse_timerange(args.timerange) + timerange = Arguments.parse_timerange(args.timerange) # Init strategy - strategy = Strategy() - strategy.init({'strategy': args.strategy}) - tick_interval = strategy.ticker_interval + try: + analyze = Analyze({'strategy': args.strategy}) + except AttributeError: + logger.critical( + 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', + args.strategy + ) + exit() + + tick_interval = analyze.strategy.ticker_interval tickers = {} if args.live: @@ -44,27 +58,32 @@ def plot_analyzed_dataframe(args) -> None: exchange._API = exchange.Bittrex({'key': '', 'secret': ''}) tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: - tickers = optimize.load_data(args.datadir, pairs=[pair], - ticker_interval=tick_interval, - refresh_pairs=False, - timerange=timerange) - dataframes = optimize.tickerdata_to_dataframe(tickers) + tickers = optimize.load_data( + datadir=args.datadir, + pairs=[pair], + ticker_interval=tick_interval, + refresh_pairs=False, + timerange=timerange + ) + dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - if (len(dataframe.index) > 750): - logger.warn('Ticker contained more than 750 candles, clipping.') - df = dataframe.tail(750) + if len(dataframe.index) > 750: + logger.warning('Ticker contained more than 750 candles, clipping.') + data = dataframe.tail(750) - candles = go.Candlestick(x=df.date, - open=df.open, - high=df.high, - low=df.low, - close=df.close, - name='Price') + candles = go.Candlestick( + x=data.date, + open=data.open, + high=data.high, + low=data.low, + close=data.close, + name='Price' + ) - df_buy = df[df['buy'] == 1] + df_buy = data[data['buy'] == 1] buys = go.Scattergl( x=df_buy.date, y=df_buy.close, @@ -73,13 +92,11 @@ def plot_analyzed_dataframe(args) -> None: marker=dict( symbol='triangle-up-dot', size=9, - line=dict( - width=1, - ), + line=dict(width=1), color='green', ) ) - df_sell = df[df['sell'] == 1] + df_sell = data[data['sell'] == 1] sells = go.Scattergl( x=df_sell.date, y=df_sell.close, @@ -88,30 +105,28 @@ def plot_analyzed_dataframe(args) -> None: marker=dict( symbol='triangle-down-dot', size=9, - line=dict( - width=1, - ), + line=dict(width=1), color='red', ) ) bb_lower = go.Scatter( - x=df.date, - y=df.bb_lowerband, + x=data.date, + y=data.bb_lowerband, name='BB lower', line={'color': "transparent"}, ) bb_upper = go.Scatter( - x=df.date, - y=df.bb_upperband, + x=data.date, + y=data.bb_upperband, name='BB upper', fill="tonexty", fillcolor="rgba(0,176,246,0.2)", line={'color': "transparent"}, ) - macd = go.Scattergl(x=df['date'], y=df['macd'], name='MACD') - macdsignal = go.Scattergl(x=df['date'], y=df['macdsignal'], name='MACD signal') - volume = go.Bar(x=df['date'], y=df['volume'], name='Volume') + macd = go.Scattergl(x=data['date'], y=data['macd'], name='MACD') + macdsignal = go.Scattergl(x=data['date'], y=data['macdsignal'], name='MACD signal') + volume = go.Bar(x=data['date'], y=data['volume'], name='Volume') fig = tools.make_subplots( rows=3, @@ -138,6 +153,31 @@ def plot_analyzed_dataframe(args) -> None: plot(fig, filename='freqtrade-plot.html') +def plot_parse_args(args): + """ + Parse args passed to the script + :param args: Cli arguments + :return: args: Array with all arguments + """ + arguments = Arguments(args, 'Graph dataframe') + arguments.scripts_options() + arguments.common_args_parser() + arguments.optimizer_shared_options(arguments.parser) + arguments.backtesting_options(arguments.parser) + + return arguments.parse_args() + + +def main(sysargv: Dict) -> None: + """ + This function will initiate the bot and start the trading loop. + :return: None + """ + logger.info('Starting Plot Dataframe') + plot_analyzed_dataframe( + plot_parse_args(sysargv) + ) + + if __name__ == '__main__': - args = plot_parse_args(sys.argv[1:]) - plot_analyzed_dataframe(args) + main(sys.argv[1:]) From 45341bb24637c43a2e3c2dd01e8d496971777b91 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 20:21:49 -0800 Subject: [PATCH 25/56] Plot_profit.py: fix it and make it works with the new object model --- config.json.bak | 44 ------------- scripts/plot_profit.py | 142 ++++++++++++++++++++++++++++++----------- 2 files changed, 104 insertions(+), 82 deletions(-) delete mode 100644 config.json.bak diff --git a/config.json.bak b/config.json.bak deleted file mode 100644 index 053e83e3f..000000000 --- a/config.json.bak +++ /dev/null @@ -1,44 +0,0 @@ -{ - "max_open_trades": 3, - "stake_currency": "BTC", - "stake_amount": 0.005, - "fiat_display_currency": "USD", - "dry_run": true, - "unfilledtimeout": 600, - "bid_strategy": { - "ask_last_balance": 0.0 - }, - "exchange": { - "name": "bittrex", - "key": "", - "secret": "", - "pair_whitelist": [ - "BTC_ETH", - "BTC_LTC", - "BTC_ETC", - "BTC_DASH", - "BTC_ZEC", - "BTC_XLM", - "BTC_NXT", - "BTC_POWR", - "BTC_ADA", - "BTC_XMR" - ], - "pair_blacklist": [ - "BTC_DOGE" - ] - }, - "experimental": { - "use_sell_signal": false, - "sell_profit_only": false - }, - "telegram": { - "enabled": true, - "token": "387056091:AAEVz29u5KwphICqGB6c63RwZjqCd7Kh6T4", - "chat_id": "391939601" - }, - "initial_state": "running", - "internals": { - "process_throttle_secs": 5 - } -} diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index c51b29309..abdfed055 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -1,31 +1,43 @@ #!/usr/bin/env python3 +""" +Script to display profits + +Mandatory Cli parameters: +-p / --pair: pair to examine + +Optional Cli parameters +-c / --config: specify configuration file +-s / --strategy: strategy to use +--timerange: specify what timerange of data to use. +""" import sys import json +from typing import Dict import numpy as np +from datetime import datetime from plotly import tools from plotly.offline import plot import plotly.graph_objs as go +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.analyze import Analyze +from freqtrade.logger import Logger + import freqtrade.optimize as optimize import freqtrade.misc as misc -from freqtrade.strategy.strategy import Strategy + +import pprint -def plot_parse_args(args): - parser = misc.common_args_parser('Graph profits') - # FIX: perhaps delete those backtesting options that are not feasible (shows up in -h) - misc.backtesting_options(parser) - misc.scripts_options(parser) - return parser.parse_args(args) +logger = Logger(name="Graph profits").get_logger() # data:: [ pair, profit-%, enter, exit, time, duration] -# data:: ['BTC_XMR', 0.00537847, '1511176800', '1511178000', 5057, 1] -# FIX: make use of the enter/exit dates to insert the -# profit more precisely into the pg array -def make_profit_array(data, px, filter_pairs=[]): +# data:: ["BTC_ETH", 0.0023975, "1515598200", "1515602100", "2018-01-10 07:30:00+00:00", 65] +def make_profit_array(data, px, min_date, interval, filter_pairs=[]): pg = np.zeros(px) # Go through the trades # and make an total profit @@ -35,10 +47,11 @@ def make_profit_array(data, px, filter_pairs=[]): if filter_pairs and pair not in filter_pairs: continue profit = trade[1] - tim = trade[4] - dur = trade[5] - ix = tim + dur - 1 + trade_sell_time = int(trade[3]) + + ix = define_index(min_date, trade_sell_time, interval) if ix < px: + logger.debug('[%s]: Add profit %s on %s', pair, profit, trade[4]) pg[ix] += profit # rewrite the pg array to go from @@ -64,47 +77,62 @@ def plot_profit(args) -> None: # We need to use the same pairs, same tick_interval # and same timeperiod as used in backtesting # to match the tickerdata against the profits-results + timerange = Arguments.parse_timerange(args.timerange) - filter_pairs = args.pair - - config = misc.load_config(args.config) - config.update({'strategy': args.strategy}) + config = Configuration(args).get_config() # Init strategy - strategy = Strategy() - strategy.init(config) + try: + analyze = Analyze({'strategy': config.get('strategy')}) + except AttributeError: + logger.critical( + 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', + config.get('strategy') + ) + exit() + # Take pairs from the cli otherwise switch to the pair in the config file + if args.pair: + filter_pairs = args.pair + filter_pairs = filter_pairs.split(',') + else: + filter_pairs = config['exchange']['pair_whitelist'] + + tick_interval = analyze.strategy.ticker_interval pairs = config['exchange']['pair_whitelist'] if filter_pairs: - filter_pairs = filter_pairs.split(',') pairs = list(set(pairs) & set(filter_pairs)) - print('Filter, keep pairs %s' % pairs) + logger.info('Filter, keep pairs %s' % pairs) - timerange = misc.parse_timerange(args.timerange) - tickers = optimize.load_data(args.datadir, pairs=pairs, - ticker_interval=strategy.ticker_interval, - refresh_pairs=False, - timerange=timerange) - dataframes = optimize.preprocess(tickers) + tickers = optimize.load_data( + datadir=args.datadir, + pairs=pairs, + ticker_interval=tick_interval, + refresh_pairs=False, + timerange=timerange + ) + dataframes = analyze.tickerdata_to_dataframe(tickers) # NOTE: the dataframes are of unequal length, # 'dates' is an merged date array of them all. dates = misc.common_datearray(dataframes) - max_x = dates.size + min_date = int(min(dates).timestamp()) + max_date = int(max(dates).timestamp()) + num_iterations = define_index(min_date, max_date, tick_interval) + 1 # Make an average close price of all the pairs that was involved. # this could be useful to gauge the overall market trend # We are essentially saying: # array <- sum dataframes[*]['close'] / num_items dataframes # FIX: there should be some onliner numpy/panda for this - avgclose = np.zeros(max_x) + avgclose = np.zeros(num_iterations) num = 0 for pair, pair_data in dataframes.items(): close = pair_data['close'] maxprice = max(close) # Normalize price to [0,1] - print('Pair %s has length %s' % (pair, len(close))) + logger.info('Pair %s has length %s' % (pair, len(close))) for x in range(0, len(close)): avgclose[x] += close[x] / maxprice # avgclose += close @@ -114,10 +142,16 @@ def plot_profit(args) -> None: # Load the profits results # And make an profits-growth array - filename = 'backtest-result.json' - with open(filename) as file: - data = json.load(file) - pg = make_profit_array(data, max_x, filter_pairs) + try: + filename = 'backtest-result.json' + with open(filename) as file: + data = json.load(file) + except FileNotFoundError: + logger.critical('File "backtest-result.json" not found. This script require backtesting ' + 'results to run.\nPlease run a backtesting with the parameter --export.') + exit(0) + + pg = make_profit_array(data, num_iterations, min_date, tick_interval, filter_pairs) # # Plot the pairs average close prices, and total profit growth @@ -128,6 +162,7 @@ def plot_profit(args) -> None: y=avgclose, name='Avg close price', ) + profit = go.Scattergl( x=dates, y=pg, @@ -140,7 +175,7 @@ def plot_profit(args) -> None: fig.append_trace(profit, 2, 1) for pair in pairs: - pg = make_profit_array(data, max_x, pair) + pg = make_profit_array(data, num_iterations, min_date, tick_interval, pair) pair_profit = go.Scattergl( x=dates, y=pg, @@ -151,6 +186,37 @@ def plot_profit(args) -> None: plot(fig, filename='freqtrade-profit-plot.html') +def define_index(min_date, max_date, interval): + """ + Return the index of a specific date + """ + return int((max_date - min_date) / (interval * 60)) + +def plot_parse_args(args): + """ + Parse args passed to the script + :param args: Cli arguments + :return: args: Array with all arguments + """ + arguments = Arguments(args, 'Graph profits') + arguments.scripts_options() + arguments.common_args_parser() + arguments.optimizer_shared_options(arguments.parser) + arguments.backtesting_options(arguments.parser) + + return arguments.parse_args() + + +def main(sysargv: Dict) -> None: + """ + This function will initiate the bot and start the trading loop. + :return: None + """ + logger.info('Starting Plot Dataframe') + plot_profit( + plot_parse_args(sysargv) + ) + + if __name__ == '__main__': - args = plot_parse_args(sys.argv[1:]) - plot_profit(args) + main(sys.argv[1:]) From 152c4483c86673395fca5dcce8ee3f08d69fa420 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 20:22:40 -0800 Subject: [PATCH 26/56] Configuration() sends a msg to user when config file not found --- freqtrade/configuration.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index c68b06dc1..fd9872eb3 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -51,8 +51,15 @@ class Configuration(object): :param path: path as str :return: configuration as dictionary """ - with open(path) as file: - conf = json.load(file) + try: + with open(path) as file: + conf = json.load(file) + except FileNotFoundError: + self.logger.critical( + 'Config file "%s" not found. Please create your config file', + path + ) + exit(0) if 'internals' not in conf: conf['internals'] = {} From cf78da5faed10fa897c75c5ffe101d476c403b70 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 20:24:01 -0800 Subject: [PATCH 27/56] Plot_profit.py: Fix Flake8 warnings --- scripts/plot_profit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index abdfed055..68434f1fb 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -15,7 +15,6 @@ import sys import json from typing import Dict import numpy as np -from datetime import datetime from plotly import tools from plotly.offline import plot @@ -29,8 +28,6 @@ from freqtrade.logger import Logger import freqtrade.optimize as optimize import freqtrade.misc as misc -import pprint - logger = Logger(name="Graph profits").get_logger() @@ -192,6 +189,7 @@ def define_index(min_date, max_date, interval): """ return int((max_date - min_date) / (interval * 60)) + def plot_parse_args(args): """ Parse args passed to the script From aa22585d4096e33c1c05c8013126518ca74b0ad5 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 23:05:44 -0800 Subject: [PATCH 28/56] Add unit test for misc.common_datearray() --- freqtrade/tests/test_misc.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index e51144496..1d2417332 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -7,7 +7,9 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock from freqtrade.analyze import Analyze -from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, file_dump_json) +from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, + common_datearray, file_dump_json) def test_shorten_date() -> None: @@ -39,6 +41,25 @@ def test_datesarray_to_datetimearray(ticker_history): assert date_len == 3 +def test_common_datearray(default_conf, mocker) -> None: + """ + Test common_datearray() + :return: None + """ + mocker.patch('freqtrade.strategy.strategy.Strategy', MagicMock()) + + analyze = Analyze(default_conf) + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) + tickerlist = {'BTC_UNITEST': tick} + dataframes = analyze.tickerdata_to_dataframe(tickerlist) + + dates = common_datearray(dataframes) + + assert dates.size == dataframes['BTC_UNITEST']['date'].size + assert dates[0] == dataframes['BTC_UNITEST']['date'][0] + assert dates[-1] == dataframes['BTC_UNITEST']['date'][-1] + + def test_file_dump_json(mocker) -> None: """ Test file_dump_json() From ba664c43415eea57f7a0373cc48e2a3668fe8053 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 4 Mar 2018 23:12:34 -0800 Subject: [PATCH 29/56] Increase Configuration._load_hyperopt_config() code coverage --- freqtrade/tests/test_configuration.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 0c532ae73..195679bfa 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -263,7 +263,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non ) -def test_hyperopt_space_argument(mocker, default_conf, caplog) -> None: +def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: """ Test setup_configuration() function """ @@ -273,6 +273,8 @@ def test_hyperopt_space_argument(mocker, default_conf, caplog) -> None: args = [ 'hyperopt', + '--epochs', '10', + '--use-mongodb', '--spaces', 'all', ] @@ -280,6 +282,16 @@ def test_hyperopt_space_argument(mocker, default_conf, caplog) -> None: configuration = Configuration(args) config = configuration.get_config() + + assert 'epochs' in config + assert int(config['epochs']) == 10 + assert log_has('Parameter --epochs detected ...', caplog.record_tuples) + assert log_has('Will run Hyperopt with for 10 epochs ...', caplog.record_tuples) + + assert 'mongodb' in config + assert config['mongodb'] is True + assert log_has('Parameter --use-mongodb detected ...', caplog.record_tuples) + assert 'spaces' in config assert config['spaces'] == ['all'] assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) From 1d43e04725f142233e2a11898c93b87b47862dcf Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 5 Mar 2018 00:11:13 -0800 Subject: [PATCH 30/56] Increase FreqtradeBot() code coverage --- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/test_freqtradebot.py | 108 +++++++++++++++++++++------ 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e245e92cb..d4c457aef 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -115,7 +115,7 @@ class FreqtradeBot(object): if new_state == State.STOPPED: time.sleep(1) elif new_state == State.RUNNING: - min_secs = self.config['internals'].get( + min_secs = self.config.get('internals', {}).get( 'process_throttle_secs', Constants.PROCESS_THROTTLE_SECS ) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 7322ba12a..521c4ba06 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -12,6 +12,7 @@ from typing import Dict, Optional import arrow import pytest import requests +import re from sqlalchemy import create_engine from freqtrade.tests.conftest import log_has @@ -110,20 +111,52 @@ def test_freqtradebot(mocker, default_conf) -> None: assert freqtrade.get_state() is State.STOPPED -@pytest.mark.skip(reason="Test not implemented") -def test_clean() -> None: +def test_clean(mocker, default_conf, caplog) -> None: """ Test clean() method """ - pass + mock_cleanup = MagicMock() + mocker.patch('freqtrade.persistence.cleanup', mock_cleanup) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.get_state() == State.RUNNING + + assert freqtrade.clean() + assert freqtrade.get_state() == State.STOPPED + assert log_has('Stopping trader and cleaning up modules...', caplog.record_tuples) + assert mock_cleanup.call_count == 1 -@pytest.mark.skip(reason="Test not implemented") -def test_worker() -> None: +def test_worker_running(mocker, default_conf, caplog) -> None: """ - Test worker() method + Test worker() method. Test when we start the bot """ - pass + mock_throttle = MagicMock() + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._throttle', mock_throttle) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + state = freqtrade.worker(old_state=None) + assert state is State.RUNNING + assert log_has('Changing state to: RUNNING', caplog.record_tuples) + assert mock_throttle.call_count == 1 + + +def test_worker_stopped(mocker, default_conf, caplog) -> None: + """ + Test worker() method. Test when we stop the bot + """ + mock_throttle = MagicMock() + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._throttle', mock_throttle) + mock_sleep = mocker.patch('time.sleep', return_value=None) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.update_state(State.STOPPED) + state = freqtrade.worker(old_state=State.RUNNING) + assert state is State.STOPPED + assert log_has('Changing state to: STOPPED', caplog.record_tuples) + assert mock_throttle.call_count == 0 + assert mock_sleep.call_count == 1 def test_throttle(mocker, default_conf, caplog) -> None: @@ -170,14 +203,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None: assert result == -1 -@pytest.mark.skip(reason="Test not implemented") -def test_process() -> None: - """ - Test _process() method - """ - pass - - def test_gen_pair_whitelist(mocker, default_conf, get_market_summaries_data) -> None: """ Test _gen_pair_whitelist() method @@ -478,14 +503,6 @@ def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, m assert result is False -@pytest.mark.skip(reason="Test not implemented") -def test_get_target_bid(): - """ - Test get_target_bid() method - """ - pass - - def test_balance_fully_ask_side(mocker) -> None: """ Test get_target_bid() method @@ -874,6 +891,51 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old assert trades[0].stake_amount == trade_buy.open_rate * trades[0].amount +def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -> None: + """ + Test check_handle_timedout() method when get_order throw an exception + """ + patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + patch_pymarketcap(mocker) + + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + handle_timedout_limit_buy=MagicMock(), + handle_timedout_limit_sell=MagicMock(), + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_buy = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + regexp = re.compile( + 'Cannot query order for Trade(id=1, pair=BTC_ETH, amount=90.99181073, ' + 'open_rate=0.00001099, open_since=10 hours ago) due to Traceback (most ' + 'recent call last):\n.*' + ) + + freqtrade.check_handle_timedout(600) + assert filter(regexp.match, caplog.record_tuples) + + def test_handle_timedout_limit_buy(mocker, default_conf) -> None: """ Test handle_timedout_limit_buy() method From ea7b25766bf5263f303b802467b5a333e456798b Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 5 Mar 2018 00:35:42 -0800 Subject: [PATCH 31/56] Increase Hyperopt() code coverage --- freqtrade/tests/optimize/test_hyperopt.py | 41 +++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 4f6b78595..e140ac672 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -3,6 +3,7 @@ import os from copy import deepcopy from unittest.mock import MagicMock import pandas as pd +from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt from freqtrade.tests.conftest import default_conf, log_has @@ -371,3 +372,43 @@ def test_has_space(): _HYPEROPT.config.update({'spaces': ['all']}) assert _HYPEROPT.has_space('buy') + + +def test_populate_indicators() -> None: + """ + Test Hyperopt.populate_indicators() + """ + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) + tickerlist = {'BTC_UNITEST': tick} + dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) + dataframe = _HYPEROPT.populate_indicators(dataframes['BTC_UNITEST']) + + # Check if some indicators are generated. We will not test all of them + assert 'adx' in dataframe + assert 'ao' in dataframe + assert 'cci' in dataframe + + +def test_buy_strategy_generator() -> None: + """ + Test Hyperopt.buy_strategy_generator() + """ + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) + tickerlist = {'BTC_UNITEST': tick} + dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) + dataframe = _HYPEROPT.populate_indicators(dataframes['BTC_UNITEST']) + + populate_buy_trend = _HYPEROPT.buy_strategy_generator( + { + 'adx': { + 'enabled': True, + 'value': 20 + }, + 'trigger': { + 'type': 'lower_bb' + } + } + ) + result = populate_buy_trend(dataframe) + # Check if some indicators are generated. We will not test all of them + assert 'adx' in result From a8fd7a69abde17c138a6cae1b944f505ae7e5d87 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 5 Mar 2018 19:44:27 -0800 Subject: [PATCH 32/56] Increase Configuration._load_config_file() code coverage --- freqtrade/tests/test_configuration.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 195679bfa..4ca92c343 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -7,6 +7,7 @@ import json from copy import deepcopy import pytest +from unittest.mock import MagicMock from jsonschema import ValidationError from freqtrade.arguments import Arguments @@ -67,6 +68,24 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: assert log_has('Validating configuration ...', caplog.record_tuples) +def test_load_config_file_exception(mocker, caplog) -> None: + """ + Test Configuration._load_config_file() method + """ + mocker.patch( + 'freqtrade.configuration.open', + MagicMock(side_effect=FileNotFoundError('File not found')) + ) + configuration = Configuration([]) + + with pytest.raises(SystemExit): + configuration._load_config_file('somefile') + assert log_has( + 'Config file "somefile" not found. Please create your config file', + caplog.record_tuples + ) + + def test_load_config(default_conf, mocker) -> None: """ Test Configuration.load_config() without any cli params From 0bb7cc8ab59c7014adcae2f2f12f68b1fea3cc45 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 5 Mar 2018 20:49:45 -0800 Subject: [PATCH 33/56] Hyperopt: fix 'Ran out of input' error --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 0d9289929..39bade554 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -524,7 +524,7 @@ class Hyperopt(Backtesting): self.logger.info('Preparing Trials..') signal.signal(signal.SIGINT, self.signal_handler) # read trials file if we have one - if os.path.exists(self.trials_file): + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: self.trials = self.read_trials() self.current_tries = len(self.trials.results) From 173b640b345958f5257c94d5d3b755ae95245dd6 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 5 Mar 2018 22:02:03 -0800 Subject: [PATCH 34/56] Increase Hyperopt() code coverage --- freqtrade/tests/optimize/test_hyperopt.py | 123 +++++++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index e140ac672..0d13646f2 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,11 +1,13 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 +import json import os from copy import deepcopy from unittest.mock import MagicMock import pandas as pd from freqtrade.optimize.__init__ import load_tickerdata_file -from freqtrade.optimize.hyperopt import Hyperopt +from freqtrade.optimize.hyperopt import Hyperopt, start from freqtrade.tests.conftest import default_conf, log_has +from freqtrade.tests.optimize.test_backtesting import get_args # Avoid to reinit the same object again and again @@ -23,6 +25,7 @@ def create_trials(mocker) -> None: _HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1) mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) @@ -39,6 +42,35 @@ def create_trials(mocker) -> None: # Unit tests +def test_start(mocker, default_conf, caplog) -> None: + """ + Test start() function + """ + start_mock = MagicMock() + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + start(args) + + import pprint + pprint.pprint(caplog.record_tuples) + + assert log_has( + 'Starting freqtrade in Hyperopt mode', + caplog.record_tuples + ) + assert start_mock.call_count == 1 + + def test_loss_calculation_prefer_correct_trade_count() -> None: """ Test Hyperopt.calculate_loss() @@ -253,7 +285,7 @@ def test_save_trials_saves_trials(mocker, caplog) -> None: mock_dump.assert_called_once() -def test_read_trials_returns_trials_file(mocker, default_conf, caplog) -> None: +def test_read_trials_returns_trials_file(mocker, caplog) -> None: trials = create_trials(mocker) mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials) mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load) @@ -400,10 +432,41 @@ def test_buy_strategy_generator() -> None: populate_buy_trend = _HYPEROPT.buy_strategy_generator( { + 'uptrend_long_ema': { + 'enabled': True + }, + 'macd_below_zero': { + 'enabled': True + }, + 'uptrend_short_ema': { + 'enabled': True + }, + 'mfi': { + 'enabled': True, + 'value': 20 + }, + 'fastd': { + 'enabled': True, + 'value': 20 + }, 'adx': { 'enabled': True, 'value': 20 }, + 'rsi': { + 'enabled': True, + 'value': 20 + }, + 'over_sar': { + 'enabled': True, + }, + 'green_candle': { + 'enabled': True, + }, + 'uptrend_sma': { + 'enabled': True, + }, + 'trigger': { 'type': 'lower_bb' } @@ -411,4 +474,58 @@ def test_buy_strategy_generator() -> None: ) result = populate_buy_trend(dataframe) # Check if some indicators are generated. We will not test all of them - assert 'adx' in result + assert 'buy' in result + assert 1 in result['buy'] + + +def test_generate_optimizer(mocker, default_conf) -> None: + """ + Test Hyperopt.generate_optimizer() function + """ + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'timerange': None}) + conf.update({'spaces': 'all'}) + + trades = [ + ('BTC_POWR', 0.023117, 0.000233, 100) + ] + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + backtest_result = pd.DataFrame.from_records(trades, columns=labels) + + mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.backtest', + MagicMock(return_value=backtest_result) + ) + + optimizer_param = { + 'adx': {'enabled': False}, + 'fastd': {'enabled': True, 'value': 35.0}, + 'green_candle': {'enabled': True}, + 'macd_below_zero': {'enabled': True}, + 'mfi': {'enabled': False}, + 'over_sar': {'enabled': False}, + 'roi_p1': 0.01, + 'roi_p2': 0.01, + 'roi_p3': 0.1, + 'roi_t1': 60.0, + 'roi_t2': 30.0, + 'roi_t3': 20.0, + 'rsi': {'enabled': False}, + 'stoploss': -0.4, + 'trigger': {'type': 'macd_cross_signal'}, + 'uptrend_long_ema': {'enabled': False}, + 'uptrend_short_ema': {'enabled': True}, + 'uptrend_sma': {'enabled': True} + } + + response_expected = { + 'loss': 1.9840569076926293, + 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' + '(0.0231Σ%). Avg duration 100.0 mins.', + 'status': 'ok' + } + + hyperopt = Hyperopt(conf) + generate_optimizer_value = hyperopt.generate_optimizer(optimizer_param) + assert generate_optimizer_value == response_expected From e6732e01e1be47bfaa95c7c0fbc5bd229966f6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rald=20LONLAS?= Date: Thu, 15 Mar 2018 15:48:22 -0700 Subject: [PATCH 35/56] Use ticker_interval defined in Strategy() instead of a mix between strategy and config file (#540) --- freqtrade/analyze.py | 7 ++ freqtrade/freqtradebot.py | 35 ++++---- freqtrade/strategy/strategy.py | 2 +- freqtrade/tests/rpc/test_rpc.py | 18 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 28 ++++--- freqtrade/tests/test_freqtradebot.py | 100 ++++++++++++----------- 6 files changed, 98 insertions(+), 92 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index c86e2dc85..401d99479 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -77,6 +77,13 @@ class Analyze(object): """ return self.strategy.populate_sell_trend(dataframe=dataframe) + def get_ticker_interval(self) -> int: + """ + Return ticker interval to use + :return: Ticker interval value to use + """ + return self.strategy.ticker_interval + def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d4c457aef..b18d6540b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -125,17 +125,9 @@ class FreqtradeBot(object): Constants.DYNAMIC_WHITELIST ) - interval = int( - self.config.get( - 'ticker_interval', - Constants.TICKER_INTERVAL - ) - ) - self._throttle(func=self._process, min_secs=min_secs, - nb_assets=nb_assets, - interval=interval) + nb_assets=nb_assets) return new_state def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: @@ -154,7 +146,7 @@ class FreqtradeBot(object): time.sleep(duration) return result - def _process(self, interval: int, nb_assets: Optional[int] = 0) -> bool: + def _process(self, nb_assets: Optional[int] = 0) -> bool: """ Queries the persistence layer for open trades and handles them, otherwise a new trade is created. @@ -179,11 +171,11 @@ class FreqtradeBot(object): # First process current opened trades for trade in trades: - state_changed |= self.process_maybe_execute_sell(trade, interval) + state_changed |= self.process_maybe_execute_sell(trade) # Then looking for buy opportunities if len(trades) < self.config['max_open_trades']: - state_changed = self.process_maybe_execute_buy(interval) + state_changed = self.process_maybe_execute_buy() if 'unfilledtimeout' in self.config: # Check and handle any timed out open orders @@ -263,9 +255,7 @@ class FreqtradeBot(object): balance = self.config['bid_strategy']['ask_last_balance'] return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) - # TODO: Remove the two parameters and use the value already in conf['stake_amount'] and - # int(conf['ticker_interval']) - def create_trade(self, stake_amount: float, interval: int) -> bool: + def create_trade(self) -> bool: """ Checks the implemented trading indicator(s) for a randomly picked pair, if one pair triggers the buy_signal a new trade record gets created @@ -273,6 +263,9 @@ class FreqtradeBot(object): :param interval: Ticker interval used for Analyze :return: True if a trade object has been created and persisted, False otherwise """ + stake_amount = self.config['stake_amount'] + interval = self.analyze.get_ticker_interval() + self.logger.info( 'Checking buy signals to create a new trade with stake_amount: %f ...', stake_amount @@ -343,14 +336,14 @@ class FreqtradeBot(object): Trade.session.flush() return True - def process_maybe_execute_buy(self, interval: int) -> bool: + def process_maybe_execute_buy(self) -> bool: """ Tries to execute a buy trade in a safe way :return: True if executed """ try: # Create entity and execute trade - if self.create_trade(float(self.config['stake_amount']), interval): + if self.create_trade(): return True self.logger.info('Found no buy signals for whitelisted currencies. Trying again..') @@ -359,7 +352,7 @@ class FreqtradeBot(object): self.logger.warning('Unable to create trade: %s', exception) return False - def process_maybe_execute_sell(self, trade: Trade, interval: int) -> bool: + def process_maybe_execute_sell(self, trade: Trade) -> bool: """ Tries to execute a sell trade :return: True if executed @@ -372,10 +365,10 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair - return self.handle_trade(trade, interval) + return self.handle_trade(trade) return False - def handle_trade(self, trade: Trade, interval: int) -> bool: + def handle_trade(self, trade: Trade) -> bool: """ Sells the current pair if the threshold is reached and updates the trade record. :return: True if trade has been sold, False otherwise @@ -389,7 +382,7 @@ class FreqtradeBot(object): (buy, sell) = (False, False) if self.config.get('experimental', {}).get('use_sell_signal'): - (buy, sell) = self.analyze.get_signal(trade.pair, interval) + (buy, sell) = self.analyze.get_signal(trade.pair, self.analyze.get_ticker_interval()) if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 607b6223f..ed1dee10e 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -64,7 +64,7 @@ class Strategy(object): # Optimal stoploss designed for the strategy self.stoploss = float(self.custom_strategy.stoploss) - self.ticker_interval = self.custom_strategy.ticker_interval + self.ticker_interval = int(self.custom_strategy.ticker_interval) def _load_strategy(self, strategy_name: str) -> None: """ diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index e9d77b1f5..764de1cb2 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -51,7 +51,7 @@ def test_rpc_trade_status(default_conf, ticker, mocker) -> None: assert error assert 'no active trade' in result - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() (error, result) = rpc.rpc_trade_status() assert not error trade = result[0] @@ -99,7 +99,7 @@ def test_rpc_status_table(default_conf, ticker, mocker) -> None: assert error assert '*Status:* `no active order`' in result - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() (error, result) = rpc.rpc_status_table() assert 'just now' in result['Since'].all() assert 'BTC_ETH' in result['Pair'].all() @@ -127,7 +127,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_s rpc = RPC(freqtradebot) # Create some test data - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -188,7 +188,7 @@ def test_rpc_trade_statistics( assert stats.find('no closed trade') >= 0 # Create some test data - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) @@ -247,7 +247,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, ticker_sell_u rpc = RPC(freqtradebot) # Create some test data - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) @@ -427,7 +427,7 @@ def test_rpc_forcesell(default_conf, ticker, mocker) -> None: assert not error assert res == '' - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() (error, res) = rpc.rpc_forcesell('all') assert not error assert res == '' @@ -462,7 +462,7 @@ def test_rpc_forcesell(default_conf, ticker, mocker) -> None: assert res == '' assert cancel_order_mock.call_count == 1 - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() # make an limit-sell open trade mocker.patch( 'freqtrade.freqtradebot.exchange.get_order', @@ -497,7 +497,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, rpc = RPC(freqtradebot) # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -540,7 +540,7 @@ def test_rpc_count(mocker, default_conf, ticker) -> None: assert nb_trades == 0 # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() (error, trades) = rpc.rpc_count() nb_trades = len(trades) assert not error diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index d227bd9fb..18764c604 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -266,7 +266,7 @@ def test_status(default_conf, update, mocker, ticker) -> None: # Create some test data for _ in range(3): - freqtradebot.create_trade(0.001, 5) + freqtradebot.create_trade() telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 3 @@ -314,7 +314,7 @@ def test_status_handle(default_conf, update, ticker, mocker) -> None: msg_mock.reset_mock() # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() # Trigger status while we have a fulfilled order for the open trade telegram._status(bot=MagicMock(), update=update) @@ -342,7 +342,9 @@ def test_status_table_handle(default_conf, update, ticker, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + conf = deepcopy(default_conf) + conf['stake_amount'] = 15.0 + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) freqtradebot.update_state(State.STOPPED) @@ -358,7 +360,7 @@ def test_status_table_handle(default_conf, update, ticker, mocker) -> None: msg_mock.reset_mock() # Create some test data - freqtradebot.create_trade(15.0, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() telegram._status_table(bot=MagicMock(), update=update) @@ -399,7 +401,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -426,8 +428,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, # Reset msg_mock msg_mock.reset_mock() # Add two other trades - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() + freqtradebot.create_trade() trades = Trade.query.all() for trade in trades: @@ -512,7 +514,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, msg_mock.reset_mock() # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade @@ -764,7 +766,7 @@ def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker) telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -803,7 +805,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() # Decrease the price and sell it mocker.patch.multiple( @@ -847,7 +849,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, mocker) -> None: # Create some test data for _ in range(4): - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() rpc_mock.reset_mock() update.message.text = '/forcesell all' @@ -925,7 +927,7 @@ def test_performance_handle(default_conf, update, ticker, limit_buy_order, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -995,7 +997,7 @@ def test_count_handle(default_conf, update, ticker, mocker) -> None: freqtradebot.update_state(State.RUNNING) # Create some test data - freqtradebot.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() msg_mock.reset_mock() telegram._count(bot=MagicMock(), update=update) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 521c4ba06..1987f62b0 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -255,7 +255,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, mocker) -> None: # Save state of current whitelist whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() assert trade is not None @@ -287,12 +287,14 @@ def test_create_trade_minimal_amount(default_conf, ticker, mocker) -> None: get_ticker=ticker, buy=buy_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) - min_stake_amount = 0.0005 - freqtrade.create_trade(min_stake_amount, int(default_conf['ticker_interval'])) + conf = deepcopy(default_conf) + conf['stake_amount'] = 0.0005 + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade() rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] - assert rate * amount >= min_stake_amount + assert rate * amount >= conf['stake_amount'] def test_create_trade_no_stake_amount(default_conf, ticker, mocker) -> None: @@ -312,7 +314,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, mocker) -> None: freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade.create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) + freqtrade.create_trade() def test_create_trade_no_pairs(default_conf, ticker, mocker) -> None: @@ -334,10 +336,10 @@ def test_create_trade_no_pairs(default_conf, ticker, mocker) -> None: conf['exchange']['pair_blacklist'] = ["BTC_ETH"] freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() with pytest.raises(DependencyException, match=r'.*No currency pairs in whitelist.*'): - freqtrade.create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) + freqtrade.create_trade() def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker) -> None: @@ -359,10 +361,10 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker) -> conf['exchange']['pair_blacklist'] = ["BTC_ETH"] freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() with pytest.raises(DependencyException, match=r'.*No currency pairs in whitelist.*'): - freqtrade.create_trade(conf['stake_amount'], int(conf['ticker_interval'])) + freqtrade.create_trade() def test_create_trade_no_signal(default_conf, mocker) -> None: @@ -381,12 +383,14 @@ def test_create_trade_no_signal(default_conf, mocker) -> None: get_ticker_history=MagicMock(return_value=20), get_balance=MagicMock(return_value=20) ) + + conf = deepcopy(default_conf) + conf['stake_amount'] = 10 freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - stake_amount = 10 Trade.query = MagicMock() Trade.query.filter = MagicMock() - assert not freqtrade.create_trade(stake_amount, int(conf['ticker_interval'])) + assert not freqtrade.create_trade() def test_process_trade_creation(default_conf, ticker, limit_buy_order, @@ -410,7 +414,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades - result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + result = freqtrade._process() assert result is True trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -447,7 +451,7 @@ def test_process_exchange_failures(default_conf, ticker, health, mocker) -> None sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) - result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + result = freqtrade._process() assert result is False assert sleep_mock.has_calls() @@ -469,7 +473,7 @@ def test_process_operational_exception(default_conf, ticker, health, mocker) -> freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) assert freqtrade.get_state() == State.RUNNING - result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + result = freqtrade._process() assert result is False assert freqtrade.get_state() == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0] @@ -494,12 +498,12 @@ def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, m trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades - result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + result = freqtrade._process() assert result is True trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert len(trades) == 1 - result = freqtrade._process(interval=int(default_conf['ticker_interval'])) + result = freqtrade._process() assert result is False @@ -537,10 +541,10 @@ def test_process_maybe_execute_buy(mocker, default_conf) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=True)) - assert freqtrade.process_maybe_execute_buy(int(default_conf['ticker_interval'])) + assert freqtrade.process_maybe_execute_buy() mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=False)) - assert not freqtrade.process_maybe_execute_buy(int(default_conf['ticker_interval'])) + assert not freqtrade.process_maybe_execute_buy() def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> None: @@ -553,7 +557,7 @@ def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> No 'freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(side_effect=DependencyException) ) - freqtrade.process_maybe_execute_buy(int(default_conf['ticker_interval'])) + freqtrade.process_maybe_execute_buy() log_has('Unable to create trade:', caplog.record_tuples) @@ -568,11 +572,11 @@ def test_process_maybe_execute_sell(mocker, default_conf) -> None: trade = MagicMock() trade.open_order_id = '123' - assert not freqtrade.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) + assert not freqtrade.process_maybe_execute_sell(trade) trade.is_open = True trade.open_order_id = None # Assert we call handle_trade() if trade is feasible for execution - assert freqtrade.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) + assert freqtrade.process_maybe_execute_sell(trade) def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker) -> None: @@ -597,7 +601,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() assert trade @@ -606,7 +610,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker) - assert trade.is_open is True patch_get_signal(mocker, value=(False, True)) - assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True + assert freqtrade.handle_trade(trade) is True assert trade.open_order_id == 'mocked_limit_sell' # Simulate fulfilled LIMIT_SELL order for trade @@ -638,7 +642,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() # Buy and Sell triggering, so doing nothing ... trades = Trade.query.all() @@ -647,7 +651,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: # Buy is triggering, so buying ... patch_get_signal(mocker, value=(True, False)) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trades = Trade.query.all() nb_trades = len(trades) assert nb_trades == 1 @@ -655,7 +659,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: # Buy and Sell are not triggering, so doing nothing ... patch_get_signal(mocker, value=(False, False)) - assert freqtrade.handle_trade(trades[0], int(default_conf['ticker_interval'])) is False + assert freqtrade.handle_trade(trades[0]) is False trades = Trade.query.all() nb_trades = len(trades) assert nb_trades == 1 @@ -663,7 +667,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: # Buy and Sell are triggering, so doing nothing ... patch_get_signal(mocker, value=(True, True)) - assert freqtrade.handle_trade(trades[0], int(default_conf['ticker_interval'])) is False + assert freqtrade.handle_trade(trades[0]) is False trades = Trade.query.all() nb_trades = len(trades) assert nb_trades == 1 @@ -672,7 +676,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: # Sell is triggering, guess what : we are Selling! patch_get_signal(mocker, value=(False, True)) trades = Trade.query.all() - assert freqtrade.handle_trade(trades[0], int(default_conf['ticker_interval'])) is True + assert freqtrade.handle_trade(trades[0]) is True def test_handle_trade_roi(default_conf, ticker, mocker, caplog) -> None: @@ -695,7 +699,7 @@ def test_handle_trade_roi(default_conf, ticker, mocker, caplog) -> None: mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() trade.is_open = True @@ -706,7 +710,7 @@ def test_handle_trade_roi(default_conf, ticker, mocker, caplog) -> None: # executing # if ROI is reached we must sell patch_get_signal(mocker, value=(False, True)) - assert freqtrade.handle_trade(trade, interval=int(default_conf['ticker_interval'])) + assert freqtrade.handle_trade(trade) assert log_has('Required profit reached. Selling..', caplog.record_tuples) @@ -730,16 +734,16 @@ def test_handle_trade_experimental(default_conf, ticker, mocker, caplog) -> None mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() trade.is_open = True patch_get_signal(mocker, value=(False, False)) - assert not freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) + assert not freqtrade.handle_trade(trade) patch_get_signal(mocker, value=(False, True)) - assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) + assert freqtrade.handle_trade(trade) assert log_has('Sell signal received. Selling..', caplog.record_tuples) @@ -759,7 +763,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) # Create trade and sell it - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() assert trade @@ -769,7 +773,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo assert trade.is_open is False with pytest.raises(ValueError, match=r'.*closed trade.*'): - freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) + freqtrade.handle_trade(trade) def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker) -> None: @@ -1004,7 +1008,7 @@ def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker) -> None: freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) # Create some test data - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() assert trade @@ -1044,7 +1048,7 @@ def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker) -> No freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) # Create some test data - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() assert trade @@ -1082,7 +1086,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, ticker_sell_up, freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) # Create some test data - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() assert trade @@ -1122,7 +1126,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) # Create some test data - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() assert trade @@ -1168,12 +1172,12 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, mocker) - 'sell_profit_only': True, } freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) patch_get_signal(mocker, value=(False, True)) - assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True + assert freqtrade.handle_trade(trade) is True def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker) -> None: @@ -1200,12 +1204,12 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker) 'sell_profit_only': False, } freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) patch_get_signal(mocker, value=(False, True)) - assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True + assert freqtrade.handle_trade(trade) is True def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker) -> None: @@ -1232,12 +1236,12 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker) -> 'sell_profit_only': True, } freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) patch_get_signal(mocker, value=(False, True)) - assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is False + assert freqtrade.handle_trade(trade) is False def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker) -> None: @@ -1266,9 +1270,9 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker) -> } freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) - freqtrade.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) patch_get_signal(mocker, value=(False, True)) - assert freqtrade.handle_trade(trade, int(default_conf['ticker_interval'])) is True + assert freqtrade.handle_trade(trade) is True From 5ed6f7001014b0f2ba98bbd26c0880e86c440ff5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 18 Mar 2018 01:55:43 +0100 Subject: [PATCH 36/56] call set_loggers() and pass sys.argv to main --- bin/freqtrade | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/freqtrade b/bin/freqtrade index 7476152fd..e7ae7a4ca 100755 --- a/bin/freqtrade +++ b/bin/freqtrade @@ -1,4 +1,7 @@ #!/usr/bin/env python3 -from freqtrade.main import main -main() \ No newline at end of file +import sys + +from freqtrade.main import main, set_loggers +set_loggers() +main(sys.argv[1:]) From 94caf82ab2666926762ab3564fa683eb33f2a9e4 Mon Sep 17 00:00:00 2001 From: Matthias <5024695+xmatthias@users.noreply.github.com> Date: Thu, 15 Mar 2018 23:37:34 +0100 Subject: [PATCH 37/56] Fix test_dataframe when ran standalone (#546) * Fix dataframe test when ran standalone * Fix standalone tests in hyperopt and optimize tests --- freqtrade/tests/optimize/test_hyperopt.py | 8 ++++++-- freqtrade/tests/test_dataframe.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 0d13646f2..047c42972 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -4,8 +4,10 @@ import os from copy import deepcopy from unittest.mock import MagicMock import pandas as pd + from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start +from freqtrade.strategy.strategy import Strategy from freqtrade.tests.conftest import default_conf, log_has from freqtrade.tests.optimize.test_backtesting import get_args @@ -59,6 +61,7 @@ def test_start(mocker, default_conf, caplog) -> None: '--epochs', '5' ] args = get_args(args) + Strategy({'strategy': 'default_strategy'}) start(args) import pprint @@ -169,7 +172,7 @@ def test_fmin_best_results(mocker, default_conf, caplog) -> None: hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() - + Strategy({'strategy': 'default_strategy'}) hyperopt.start() exists = [ @@ -210,7 +213,7 @@ def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None: conf.update({'spaces': 'all'}) mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) - + Strategy({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() @@ -253,6 +256,7 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> No mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + Strategy({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = trials hyperopt.tickerdata_to_dataframe = MagicMock() diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index eaa761bdb..86944e4f3 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,8 +1,10 @@ # pragma pylint: disable=missing-docstring, C0103 import pandas + from freqtrade.optimize import load_data from freqtrade.analyze import Analyze +from freqtrade.strategy.strategy import Strategy _pairs = ['BTC_ETH'] @@ -19,11 +21,13 @@ def load_dataframe_pair(pairs): def test_dataframe_load(): + Strategy({'strategy': 'default_strategy'}) dataframe = load_dataframe_pair(_pairs) assert isinstance(dataframe, pandas.core.frame.DataFrame) def test_dataframe_columns_exists(): + Strategy({'strategy': 'default_strategy'}) dataframe = load_dataframe_pair(_pairs) assert 'high' in dataframe.columns assert 'low' in dataframe.columns From b67257db356e2adac3375529e4489d3969ba1a21 Mon Sep 17 00:00:00 2001 From: Matthias <5024695+xmatthias@users.noreply.github.com> Date: Sun, 18 Mar 2018 00:42:24 +0100 Subject: [PATCH 38/56] replace pymarketcap with coinmarketcap (#562) * replace pymarketcap with coinmarketcap * fix tests to use coinmarketcap instead of pymarketcap * use arraypos 0 * update setup.py from pymarketcap to coinmarketcap * Add test to check for unsupported Crypto currency --- freqtrade/fiat_convert.py | 18 ++++-- freqtrade/tests/conftest.py | 2 +- freqtrade/tests/rpc/test_rpc.py | 27 ++++----- freqtrade/tests/rpc/test_rpc_telegram.py | 54 +++++++++--------- freqtrade/tests/test_fiat_convert.py | 7 ++- freqtrade/tests/test_freqtradebot.py | 73 ++++++++++++------------ freqtrade/tests/test_main.py | 1 - requirements.txt | 2 +- setup.py | 2 +- 9 files changed, 97 insertions(+), 89 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 6f9d3d3d5..4874247db 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -5,7 +5,7 @@ e.g BTC to USD import logging import time -from pymarketcap import Pymarketcap +from coinmarketcap import Market logger = logging.getLogger(__name__) @@ -72,11 +72,17 @@ class CryptoToFiatConverter(object): "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" ] + CRYPTOMAP = { + 'BTC': 'bitcoin', + 'ETH': 'ethereum', + 'USDT': 'thether' + } + def __new__(cls): if CryptoToFiatConverter.__instance is None: CryptoToFiatConverter.__instance = object.__new__(cls) try: - CryptoToFiatConverter._coinmarketcap = Pymarketcap() + CryptoToFiatConverter._coinmarketcap = Market() except BaseException: CryptoToFiatConverter._coinmarketcap = None return CryptoToFiatConverter.__instance @@ -171,12 +177,16 @@ class CryptoToFiatConverter(object): # Check if the fiat convertion you want is supported if not self._is_supported_fiat(fiat=fiat_symbol): raise ValueError('The fiat {} is not supported.'.format(fiat_symbol)) + + if crypto_symbol not in self.CRYPTOMAP: + raise ValueError( + 'The crypto symbol {} is not supported.'.format(crypto_symbol)) try: return float( self._coinmarketcap.ticker( - currency=crypto_symbol, + currency=self.CRYPTOMAP[crypto_symbol], convert=fiat_symbol - )['price_' + fiat_symbol.lower()] + )[0]['price_' + fiat_symbol.lower()] ) except BaseException: return 0.0 diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 60c4f99ed..cb726633b 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -33,7 +33,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: :param config: Config to pass to the bot :return: None """ - mocker.patch('freqtrade.fiat_convert.Pymarketcap', {'price_usd': 12345.0}) + mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0}) mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 764de1cb2..50943b1bc 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -13,7 +13,7 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC from freqtrade.state import State -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_pymarketcap +from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap # Functions for recurrent object patching @@ -30,7 +30,7 @@ def test_rpc_trade_status(default_conf, ticker, mocker) -> None: Test rpc_trade_status() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -78,7 +78,7 @@ def test_rpc_status_table(default_conf, ticker, mocker) -> None: Test rpc_status_table() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -112,7 +112,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_s Test rpc_daily_profit() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -165,9 +165,8 @@ def test_rpc_trade_statistics( """ patch_get_signal(mocker, (True, False)) mocker.patch.multiple( - 'freqtrade.fiat_convert.Pymarketcap', + 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1}) ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) @@ -228,9 +227,8 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, ticker_sell_u """ patch_get_signal(mocker, (True, False)) mocker.patch.multiple( - 'freqtrade.fiat_convert.Pymarketcap', + 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1}) ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) @@ -303,9 +301,8 @@ def test_rpc_balance_handle(default_conf, mocker): patch_get_signal(mocker, (True, False)) mocker.patch.multiple( - 'freqtrade.fiat_convert.Pymarketcap', + 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), - _cache_symbols=MagicMock(return_value={'BTC': 1}) ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) @@ -337,7 +334,7 @@ def test_rpc_start(mocker, default_conf) -> None: Test rpc_start() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -365,7 +362,7 @@ def test_rpc_stop(mocker, default_conf) -> None: Test rpc_stop() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -393,7 +390,7 @@ def test_rpc_forcesell(default_conf, ticker, mocker) -> None: Test rpc_forcesell() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) cancel_order_mock = MagicMock() @@ -484,7 +481,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, Test rpc_performance() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -522,7 +519,7 @@ def test_rpc_count(mocker, default_conf, ticker) -> None: Test rpc_count() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 18764c604..69ddc5ef5 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -21,7 +21,7 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.rpc.telegram import Telegram from freqtrade.persistence import Trade from freqtrade.state import State -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_pymarketcap +from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap from freqtrade.tests.conftest import get_patched_freqtradebot, log_has @@ -147,7 +147,7 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: Test authorized_only() method when we are authorized """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) chat = Chat(0, 0) @@ -178,7 +178,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: Test authorized_only() method when we are unauthorized """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) chat = Chat(0xdeadbeef, 0) @@ -209,7 +209,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: Test authorized_only() method when an exception is thrown """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) update = Update(randint(1, 100)) @@ -244,7 +244,7 @@ def test_status(default_conf, update, mocker, ticker) -> None: conf['telegram']['chat_id'] = 123 patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -282,7 +282,7 @@ def test_status_handle(default_conf, update, ticker, mocker) -> None: Test _status() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -327,7 +327,7 @@ def test_status_table_handle(default_conf, update, ticker, mocker) -> None: Test _status_table() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -379,7 +379,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, Test _daily() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch( 'freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0 @@ -451,7 +451,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: Test _daily() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -490,7 +490,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, Test _profit() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -597,7 +597,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: } patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=mock_balance) mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', side_effect=mock_ticker) @@ -628,7 +628,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: Test _balance() method when the Exchange platform returns nothing """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=[]) @@ -652,7 +652,7 @@ def test_start_handle(default_conf, update, mocker) -> None: """ Test _start() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( @@ -676,7 +676,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: """ Test _start() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( @@ -701,7 +701,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: """ Test _stop() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( @@ -726,7 +726,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: """ Test _stop() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( @@ -752,7 +752,7 @@ def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker) Test _forcesell() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) @@ -791,7 +791,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m Test _forcesell() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) @@ -834,7 +834,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, mocker) -> None: Test _forcesell() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) @@ -867,7 +867,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: Test _forcesell() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) msg_mock = MagicMock() mocker.patch.multiple( @@ -910,7 +910,7 @@ def test_performance_handle(default_conf, update, ticker, limit_buy_order, Test _performance() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -950,7 +950,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: Test _performance() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -973,7 +973,7 @@ def test_count_handle(default_conf, update, ticker, mocker) -> None: Test _count() method """ patch_get_signal(mocker, (True, False)) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -1011,7 +1011,7 @@ def test_help_handle(default_conf, update, mocker) -> None: """ Test _help() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( @@ -1031,7 +1031,7 @@ def test_version_handle(default_conf, update, mocker) -> None: """ Test _version() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( @@ -1051,7 +1051,7 @@ def test_send_msg(default_conf, mocker) -> None: """ Test send_msg() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) @@ -1073,7 +1073,7 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None: """ Test send_msg() method """ - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 7d0acfc91..49a2adc05 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -71,12 +71,15 @@ def test_fiat_convert_find_price(mocker): 'price_usd': 12345.0, 'price_eur': 13000.2 }) - mocker.patch('freqtrade.fiat_convert.Pymarketcap.ticker', api_mock) + mocker.patch('freqtrade.fiat_convert.Market.ticker', api_mock) fiat_convert = CryptoToFiatConverter() with pytest.raises(ValueError, match=r'The fiat ABC is not supported.'): fiat_convert._find_price(crypto_symbol='BTC', fiat_symbol='ABC') + with pytest.raises(ValueError, match=r'The crypto symbol XRP is not supported.'): + fiat_convert.get_price(crypto_symbol='XRP', fiat_symbol='USD') + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=12345.0) assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 12345.0 assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 12345.0 @@ -90,7 +93,7 @@ def test_fiat_convert_get_price(mocker): 'price_usd': 28000.0, 'price_eur': 15000.0 }) - mocker.patch('freqtrade.fiat_convert.Pymarketcap.ticker', api_mock) + mocker.patch('freqtrade.fiat_convert.Market.ticker', api_mock) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=28000.0) fiat_convert = CryptoToFiatConverter() diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1987f62b0..5d9b9aba1 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -35,7 +35,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) return FreqtradeBot(config, create_engine('sqlite://')) @@ -64,18 +64,18 @@ def patch_RPCManager(mocker) -> MagicMock: return rpc_mock -def patch_pymarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: +def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: """ - Mocker to Pymarketcap to speed up tests - :param mocker: mocker to patch Pymarketcap class + Mocker to coinmarketcap to speed up tests + :param mocker: mocker to patch coinmarketcap class :return: None """ - pymarketcap = MagicMock() + mock = MagicMock() if value: - pymarketcap.ticker = {'price_usd': 12345.0} + mock.ticker = {'price_usd': 12345.0} - mocker.patch('freqtrade.fiat_convert.Pymarketcap', pymarketcap) + mocker.patch('freqtrade.fiat_convert.Market', mock) # Unit tests @@ -244,7 +244,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, mocker) -> None: """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -279,7 +279,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, mocker) -> None: """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value='mocked_limit_buy') mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -303,7 +303,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, mocker) -> None: """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -323,7 +323,7 @@ def test_create_trade_no_pairs(default_conf, ticker, mocker) -> None: """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -348,7 +348,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker) -> """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -376,7 +376,7 @@ def test_create_trade_no_signal(default_conf, mocker) -> None: patch_get_signal(mocker, value=(False, False)) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -400,7 +400,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -440,7 +440,7 @@ def test_process_exchange_failures(default_conf, ticker, health, mocker) -> None """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -462,7 +462,7 @@ def test_process_operational_exception(default_conf, ticker, health, mocker) -> """ patch_get_signal(mocker) msg_mock = patch_RPCManager(mocker) - patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -485,7 +485,7 @@ def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, m """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -596,8 +596,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker) - buy=MagicMock(return_value='mocked_limit_buy'), sell=MagicMock(return_value='mocked_limit_sell') ) - patch_pymarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.fiat_convert.Pymarketcap._cache_symbols', return_value={'BTC': 1}) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) @@ -631,7 +630,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: patch_get_signal(mocker, value=(True, True)) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -689,7 +688,7 @@ def test_handle_trade_roi(default_conf, ticker, mocker, caplog) -> None: patch_get_signal(mocker, value=(True, False)) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -724,7 +723,7 @@ def test_handle_trade_experimental(default_conf, ticker, mocker, caplog) -> None patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -753,7 +752,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -782,7 +781,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mo """ rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -820,7 +819,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, Test check_handle_timedout() method """ rpc_mock = patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -859,7 +858,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old Test check_handle_timedout() method """ rpc_mock = patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -901,7 +900,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - """ patch_RPCManager(mocker) cancel_order_mock = MagicMock() - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', @@ -945,7 +944,7 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None: Test handle_timedout_limit_buy() method """ patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -972,7 +971,7 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: """ patch_RPCManager(mocker) cancel_order_mock = MagicMock() - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -998,7 +997,7 @@ def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker) -> None: """ patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -1038,7 +1037,7 @@ def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker) -> No """ patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -1077,7 +1076,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, ticker_sell_up, """ patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) - patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -1117,7 +1116,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, """ patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) - patch_pymarketcap(mocker, value={'price_usd': 12345.0}) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -1154,7 +1153,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, mocker) - """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -1186,7 +1185,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker) """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -1218,7 +1217,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker) -> """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -1250,7 +1249,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker) -> """ patch_get_signal(mocker) patch_RPCManager(mocker) - patch_pymarketcap(mocker) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index d1bade6ae..83da04495 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -72,7 +72,6 @@ def test_main(mocker, caplog) -> None: ), clean=MagicMock(), ) - args = ['-c', 'config.json.example'] # Test Main + the KeyboardInterrupt exception diff --git a/requirements.txt b/requirements.txt index 0212b91e5..646e8b8ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.158 +coinmarketcap==4.2.1 # Required for plotting data #plotly==2.3.0 diff --git a/setup.py b/setup.py index e53606dea..4b0635efa 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ setup(name='freqtrade', 'TA-Lib', 'tabulate', 'cachetools', - 'pymarketcap', + 'coinmarketcap', ], include_package_data=True, zip_safe=False, From 93931eb32b1b6936f7cb651c3048810a6511faaa Mon Sep 17 00:00:00 2001 From: gcarq Date: Mon, 19 Mar 2018 23:05:12 +0100 Subject: [PATCH 39/56] fix typo in _generate_text_table --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 92591f98d..f847b98d8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,7 +95,7 @@ class Backtesting(object): len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), - result.duration.mean(), + results.duration.mean(), len(results[results.profit_BTC > 0]), len(results[results.profit_BTC < 0]) ]) From a6a38735b1c079c95786f29a29eae2fcd3a84172 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 20 Mar 2018 19:38:33 +0100 Subject: [PATCH 40/56] backtesting: only respect max_open_trades with realistic_simulation --- freqtrade/optimize/backtesting.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f847b98d8..8d22d00e3 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -227,7 +227,13 @@ class Backtesting(object): timerange=timerange ) - max_open_trades = self.config.get('max_open_trades', 0) + # Ignore max_open_trades in backtesting, except realistic flag was passed + if self.config.get('realistic_simulation', False): + max_open_trades = self.config['max_open_trades'] + else: + self.logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') + max_open_trades = 0 + preprocessed = self.tickerdata_to_dataframe(data) # Print timeframe From cae7be44472a2d37be36e844d0aae6c049e27e23 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:12:21 +0100 Subject: [PATCH 41/56] add fee param to function doc --- freqtrade/persistence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index e0d433a24..56fe336d2 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -207,6 +207,7 @@ class Trade(_DECL_BASE): Calculates the profit in percentage (including fee). :param rate: rate to compare with (optional). If rate is not set self.close_rate will be used + :param fee: fee to use on the close rate (optional). :return: profit in percentage as float """ getcontext().prec = 8 From 31e2aa0f38ab79d4371bdab9f9baa40dbb1eb8ee Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:12:42 +0100 Subject: [PATCH 42/56] misc: apply missing typehints --- freqtrade/misc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 814e0fb6f..d70364999 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -6,12 +6,15 @@ import re import json import logging from datetime import datetime +from typing import Dict + import numpy as np +from pandas import DataFrame logger = logging.getLogger(__name__) -def shorten_date(_date): +def shorten_date(_date: str) -> str: """ Trim the date so it fits on small screens """ @@ -28,7 +31,7 @@ def shorten_date(_date): # Matplotlib doesn't support ::datetime64, # # so we need to convert it into ::datetime # ############################################ -def datesarray_to_datetimearray(dates): +def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: """ Convert an pandas-array of timestamps into An numpy-array of datetimes @@ -42,10 +45,10 @@ def datesarray_to_datetimearray(dates): return np.array(times) -def common_datearray(dfs): +def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray: """ Return dates from Dataframe - :param dfs: Dataframe + :param dfs: Dict with format pair: pair_data :return: List of dates """ alldates = {} From 2de63133ae57f4bacbd9506456beff9d83d40235 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:16:03 +0100 Subject: [PATCH 43/56] indicator_helpers: apply correct typehints --- freqtrade/indicator_helpers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/indicator_helpers.py b/freqtrade/indicator_helpers.py index c3cc42e6a..14519d7a2 100644 --- a/freqtrade/indicator_helpers.py +++ b/freqtrade/indicator_helpers.py @@ -1,19 +1,19 @@ from math import exp, pi, sqrt, cos -import numpy +import numpy as np import talib as ta from pandas import Series -def went_up(series: Series) -> Series: +def went_up(series: Series) -> bool: return series > series.shift(1) -def went_down(series: Series) -> Series: +def went_down(series: Series) -> bool: return series < series.shift(1) -def ehlers_super_smoother(series: Series, smoothing: float = 6): +def ehlers_super_smoother(series: Series, smoothing: float = 6) -> type(Series): magic = pi * sqrt(2) / smoothing a1 = exp(-magic) coeff2 = 2 * a1 * cos(magic) @@ -29,7 +29,7 @@ def ehlers_super_smoother(series: Series, smoothing: float = 6): return filtered -def fishers_inverse(series: Series, smoothing: float = 0): +def fishers_inverse(series: Series, smoothing: float = 0) -> np.ndarray: """ Does a smoothed fishers inverse transformation. Can be used with any oscillator that goes from 0 to 100 like RSI or MFI """ v1 = 0.1 * (series - 50) @@ -37,4 +37,4 @@ def fishers_inverse(series: Series, smoothing: float = 0): v2 = ta.WMA(v1.values, timeperiod=smoothing) else: v2 = v1 - return (numpy.exp(2 * v2)-1) / (numpy.exp(2 * v2) + 1) + return (np.exp(2 * v2)-1) / (np.exp(2 * v2) + 1) From 90be78b28353764a727eaecb3994c5e006d77dd3 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:18:05 +0100 Subject: [PATCH 44/56] CryptoFiat: inherit from object explicitly --- freqtrade/fiat_convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 4874247db..f73de4bec 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -10,7 +10,7 @@ from coinmarketcap import Market logger = logging.getLogger(__name__) -class CryptoFiat(): +class CryptoFiat(object): """ Object to describe what is the price of Crypto-currency in a FIAT """ From 1074415d30105010dedf1aa3e5f6e5741135c68b Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:18:25 +0100 Subject: [PATCH 45/56] remove invalid typehint from ctor --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b18d6540b..08ef74bbd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -27,7 +27,7 @@ class FreqtradeBot(object): This is from here the bot start its logic. """ - def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None) -> bool: + def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None): """ Init all variables and object the bot need to work :param config: configuration dict, you can use the Configuration.get_config() From ed71340a904b2a4d052cea971a8d293cd663a6e4 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:20:45 +0100 Subject: [PATCH 46/56] arguments: apply missing typehints --- freqtrade/arguments.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 30f94fddc..c53fd42ed 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -6,7 +6,7 @@ import argparse import os import re import logging -from typing import List +from typing import List, Tuple, Optional from freqtrade import __version__ from freqtrade.constants import Constants @@ -22,7 +22,7 @@ class Arguments(object): self.parsed_arg = None self.parser = argparse.ArgumentParser(description=description) - def _load_args(self): + def _load_args(self) -> None: self.common_args_parser() self._build_subcommands() @@ -205,7 +205,7 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: str) -> (List, int, int): + def parse_timerange(text: str) -> Optional[Tuple[List, int, int]]: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange @@ -236,10 +236,10 @@ class Arguments(object): stop = rvals[index] if stype[1] != 'date': stop = int(stop) - return (stype, start, stop) + return stype, start, stop raise Exception('Incorrect syntax for timerange "%s"' % text) - def scripts_options(self): + def scripts_options(self) -> None: """ Parses given arguments for plot scripts. """ From 5532cedcdd30b1bb552448a326fd4efb8d202a79 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:22:35 +0100 Subject: [PATCH 47/56] get_signal: remove redundant parentheses --- freqtrade/analyze.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 401d99479..fd3813458 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -3,7 +3,7 @@ Functions to analyze ticker data with indicators and produce buy and sell signal """ from datetime import datetime, timedelta from enum import Enum -from typing import Dict, List +from typing import Dict, List, Tuple import arrow from pandas import DataFrame, to_datetime from freqtrade.exchange import get_ticker_history @@ -96,9 +96,7 @@ class Analyze(object): dataframe = self.populate_sell_trend(dataframe) return dataframe - # FIX: Maybe return False, if an error has occured, - # Otherwise we might mask an error as an non-signal-scenario - def get_signal(self, pair: str, interval: int) -> (bool, bool): + def get_signal(self, pair: str, interval: int) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format BTC_ANT or BTC-ANT @@ -108,7 +106,7 @@ class Analyze(object): ticker_hist = get_ticker_history(pair, interval) if not ticker_hist: self.logger.warning('Empty ticker history for pair %s', pair) - return (False, False) # return False ? + return False, False try: dataframe = self.analyze_ticker(ticker_hist) @@ -118,18 +116,18 @@ class Analyze(object): pair, str(error) ) - return (False, False) # return False ? + return False, False except Exception as error: self.logger.exception( 'Unexpected error when analyzing ticker for pair %s: %s', pair, str(error) ) - return (False, False) # return False ? + return False, False if dataframe.empty: self.logger.warning('Empty dataframe for pair %s', pair) - return (False, False) # return False ? + return False, False latest = dataframe.iloc[-1] @@ -141,7 +139,7 @@ class Analyze(object): pair, (arrow.utcnow() - signal_date).seconds // 60 ) - return (False, False) # return False ? + return False, False (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 self.logger.debug( @@ -151,7 +149,7 @@ class Analyze(object): str(buy), str(sell) ) - return (buy, sell) + return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: """ From 532753318828726a14522645e827e532cf4e8c28 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:43:36 +0100 Subject: [PATCH 48/56] optimize: set correct typehints --- freqtrade/optimize/__init__.py | 19 +++++++++++-------- freqtrade/optimize/backtesting.py | 15 +++++++++------ freqtrade/optimize/hyperopt.py | 15 ++++++++------- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 049b2f94f..1ea6b9fe5 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -2,7 +2,7 @@ import json import os -from typing import Optional, List, Dict +from typing import Optional, List, Dict, Tuple import gzip from freqtrade.exchange import get_ticker_history @@ -13,8 +13,8 @@ from user_data.hyperopt_conf import hyperopt_optimize_conf logger = Logger(name=__name__).get_logger() -def trim_tickerlist(tickerlist, timerange): - (stype, start, stop) = timerange +def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: + stype, start, stop = timerange if stype == (None, 'line'): return tickerlist[stop:] elif stype == ('line', None): @@ -25,7 +25,10 @@ def trim_tickerlist(tickerlist, timerange): return tickerlist -def load_tickerdata_file(datadir, pair, ticker_interval, timerange=None): +def load_tickerdata_file( + datadir: str, pair: str, + ticker_interval: int, + timerange: Optional[Tuple[Tuple, int, int]] = None) -> Optional[List[Dict]]: """ Load a pair from file, :return dict OR empty if unsuccesful @@ -55,12 +58,12 @@ def load_tickerdata_file(datadir, pair, ticker_interval, timerange=None): return pairdata -def load_data(datadir: str, ticker_interval: int, pairs: Optional[List[str]] = None, - refresh_pairs: Optional[bool] = False, timerange=None) -> Dict[str, List]: +def load_data(datadir: str, ticker_interval: int, + pairs: Optional[List[str]] = None, + refresh_pairs: Optional[bool] = False, + timerange: Optional[Tuple[Tuple, int, int]] = None) -> Dict[str, List]: """ Loads ticker history data for the given parameters - :param ticker_interval: ticker interval in minutes - :param pairs: list of pairs :return: dict """ result = {} diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8d22d00e3..606af02c7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -3,8 +3,8 @@ """ This module contains the backtesting logic """ - -from typing import Dict, Tuple, Any +from argparse import Namespace +from typing import Dict, Tuple, Any, List, Optional import arrow from pandas import DataFrame, Series from tabulate import tabulate @@ -101,7 +101,10 @@ class Backtesting(object): ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) - def _get_sell_trade_entry(self, pair, buy_row, partial_ticker, trade_count_lock, args): + def _get_sell_trade_entry( + self, pair: str, buy_row: DataFrame, + partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[Tuple]: + stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) trade = Trade( @@ -132,7 +135,7 @@ class Backtesting(object): sell_row.date return None - def backtest(self, args) -> DataFrame: + def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -273,7 +276,7 @@ class Backtesting(object): ) -def setup_configuration(args) -> Dict[str, Any]: +def setup_configuration(args: Namespace) -> Dict[str, Any]: """ Prepare the configuration for the backtesting :param args: Cli args from Arguments() @@ -289,7 +292,7 @@ def setup_configuration(args) -> Dict[str, Any]: return config -def start(args) -> None: +def start(args: Namespace) -> None: """ Start Backtesting script :param args: Cli args from Arguments() diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 39bade554..5175345e5 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,10 +10,11 @@ import os import pickle import signal import sys +from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, List import numpy import talib.abstract as ta @@ -240,7 +241,7 @@ class Hyperopt(Backtesting): return trade_loss + profit_loss + duration_loss @staticmethod - def generate_roi_table(params) -> Dict[int, float]: + def generate_roi_table(params: Dict) -> Dict[int, float]: """ Generate the ROI table thqt will be used by Hyperopt """ @@ -335,7 +336,7 @@ class Hyperopt(Backtesting): ]), } - def has_space(self, space) -> bool: + def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration """ @@ -433,7 +434,7 @@ class Hyperopt(Backtesting): return populate_buy_trend - def generate_optimizer(self, params) -> Dict: + def generate_optimizer(self, params: Dict) -> Dict: if self.has_space('roi'): self.analyze.strategy.minimal_roi = self.generate_roi_table(params) @@ -496,7 +497,7 @@ class Hyperopt(Backtesting): results.duration.mean(), ) - def start(self): + def start(self) -> None: timerange = Arguments.parse_timerange(self.config.get('timerange')) data = load_data( datadir=self.config.get('datadir'), @@ -571,7 +572,7 @@ class Hyperopt(Backtesting): # Store trials result to file to resume next time self.save_trials() - def signal_handler(self, sig, frame): + def signal_handler(self, sig, frame) -> None: """ Hyperopt SIGINT handler """ @@ -585,7 +586,7 @@ class Hyperopt(Backtesting): sys.exit(0) -def start(args) -> None: +def start(args: Namespace) -> None: """ Start Backtesting script :param args: Cli args from Arguments() From d8689e504517f67d39f79df8706a0dbb14f933e1 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:43:59 +0100 Subject: [PATCH 49/56] set correct typehint; remove unused argument --- freqtrade/configuration.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index fd9872eb3..10ac783e6 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -3,8 +3,9 @@ This module contains the configuration class """ import json +from argparse import Namespace -from typing import Dict, List, Any +from typing import Dict, Any from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match @@ -17,7 +18,7 @@ class Configuration(object): Class to read and init the bot configuration Reuse this class for the bot, backtesting, hyperopt and every script that required configuration """ - def __init__(self, args: List[str], do_not_init=False) -> None: + def __init__(self, args: Namespace) -> None: self.args = args self.logging = Logger(name=__name__) self.logger = self.logging.get_logger() From d2aea7bdc1c8c7637f1c0a27f4bccaee437943ab Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 22:44:47 +0100 Subject: [PATCH 50/56] optimize imports --- freqtrade/analyze.py | 4 +++- freqtrade/arguments.py | 2 +- freqtrade/configuration.py | 2 +- freqtrade/exchange/bittrex.py | 2 +- freqtrade/fiat_convert.py | 1 + freqtrade/freqtradebot.py | 8 ++++--- freqtrade/main.py | 5 +++-- freqtrade/misc.py | 2 +- freqtrade/optimize/__init__.py | 4 ++-- freqtrade/optimize/backtesting.py | 7 ++++--- freqtrade/optimize/hyperopt.py | 8 +++---- freqtrade/rpc/rpc.py | 10 +++++---- freqtrade/rpc/telegram.py | 2 ++ freqtrade/strategy/default_strategy.py | 3 ++- freqtrade/strategy/interface.py | 1 + freqtrade/strategy/strategy.py | 9 ++++---- freqtrade/tests/conftest.py | 11 +++++----- freqtrade/tests/exchange/test_exchange.py | 11 +++++----- .../tests/exchange/test_exchange_bittrex.py | 4 +++- freqtrade/tests/optimize/test_backtesting.py | 21 ++++++++++--------- freqtrade/tests/optimize/test_hyperopt.py | 1 + freqtrade/tests/optimize/test_optimize.py | 7 ++++--- freqtrade/tests/rpc/test_rpc_telegram.py | 8 +++---- .../tests/strategy/test_default_strategy.py | 4 +++- freqtrade/tests/strategy/test_strategy.py | 1 + freqtrade/tests/test_analyze.py | 4 ++-- freqtrade/tests/test_arguments.py | 1 + freqtrade/tests/test_configuration.py | 4 ++-- freqtrade/tests/test_dataframe.py | 2 +- freqtrade/tests/test_freqtradebot.py | 9 ++++---- freqtrade/tests/test_indicator_helpers.py | 1 + freqtrade/tests/test_logger.py | 1 + freqtrade/tests/test_main.py | 1 + freqtrade/tests/test_misc.py | 3 ++- freqtrade/tests/test_persistence.py | 2 ++ .../tests/testdata/download_backtest_data.py | 4 ++-- 36 files changed, 101 insertions(+), 69 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index fd3813458..8bc552d74 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -4,12 +4,14 @@ Functions to analyze ticker data with indicators and produce buy and sell signal from datetime import datetime, timedelta from enum import Enum from typing import Dict, List, Tuple + import arrow from pandas import DataFrame, to_datetime + from freqtrade.exchange import get_ticker_history from freqtrade.logger import Logger -from freqtrade.strategy.strategy import Strategy from freqtrade.persistence import Trade +from freqtrade.strategy.strategy import Strategy class SignalType(Enum): diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index c53fd42ed..a73a1945f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -3,9 +3,9 @@ This module contains the argument manager class """ import argparse +import logging import os import re -import logging from typing import List, Tuple, Optional from freqtrade import __version__ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 10ac783e6..1f6cea4e6 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -4,8 +4,8 @@ This module contains the configuration class import json from argparse import Namespace - from typing import Dict, Any + from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 5aa07e460..0cba621af 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,8 +1,8 @@ import logging from typing import Dict, List, Optional -from bittrex.bittrex import Bittrex as _Bittrex from bittrex.bittrex import API_V1_1, API_V2_0 +from bittrex.bittrex import Bittrex as _Bittrex from requests.exceptions import ContentDecodingError from freqtrade import OperationalException diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index f73de4bec..b86b56ec5 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -5,6 +5,7 @@ e.g BTC to USD import logging import time + from coinmarketcap import Market logger = logging.getLogger(__name__) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 08ef74bbd..e57f177e9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,11 +6,14 @@ import copy import json import time import traceback -from typing import Dict, List, Optional, Any, Callable from datetime import datetime -import requests +from typing import Dict, List, Optional, Any, Callable + import arrow +import requests from cachetools import cached, TTLCache + +from freqtrade import (DependencyException, OperationalException, exchange, persistence) from freqtrade.analyze import Analyze from freqtrade.constants import Constants from freqtrade.fiat_convert import CryptoToFiatConverter @@ -18,7 +21,6 @@ from freqtrade.logger import Logger from freqtrade.persistence import Trade from freqtrade.rpc.rpc_manager import RPCManager from freqtrade.state import State -from freqtrade import (DependencyException, OperationalException, exchange, persistence) class FreqtradeBot(object): diff --git a/freqtrade/main.py b/freqtrade/main.py index 97515b68b..8deeb7da6 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -7,11 +7,12 @@ Read the documentation to know what cli arguments you need. import logging import sys from typing import Dict -from freqtrade.configuration import Configuration + +from freqtrade import (__version__) from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot from freqtrade.logger import Logger -from freqtrade import (__version__) logger = Logger(name='freqtrade').get_logger() diff --git a/freqtrade/misc.py b/freqtrade/misc.py index d70364999..f5d045c44 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,9 +2,9 @@ Various tool function for Freqtrade and scripts """ -import re import json import logging +import re from datetime import datetime from typing import Dict diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 1ea6b9fe5..a26744691 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,12 +1,12 @@ # pragma pylint: disable=missing-docstring +import gzip import json import os from typing import Optional, List, Dict, Tuple -import gzip -from freqtrade.exchange import get_ticker_history from freqtrade import misc +from freqtrade.exchange import get_ticker_history from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 606af02c7..d8af47326 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -5,16 +5,17 @@ This module contains the backtesting logic """ from argparse import Namespace from typing import Dict, Tuple, Any, List, Optional + import arrow from pandas import DataFrame, Series from tabulate import tabulate import freqtrade.optimize as optimize -from freqtrade.arguments import Arguments -from freqtrade.exchange import Bittrex -from freqtrade.configuration import Configuration from freqtrade import exchange from freqtrade.analyze import Analyze +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Bittrex from freqtrade.logger import Logger from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5175345e5..7dcd46fd2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,7 +14,7 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, List +from typing import Dict, Any, Callable import numpy import talib.abstract as ta @@ -23,11 +23,11 @@ from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.configuration import Configuration -from freqtrade.optimize import load_data from freqtrade.arguments import Arguments -from freqtrade.optimize.backtesting import Backtesting +from freqtrade.configuration import Configuration from freqtrade.logger import Logger +from freqtrade.optimize import load_data +from freqtrade.optimize.backtesting import Backtesting from user_data.hyperopt_conf import hyperopt_optimize_conf diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index fa9c2022d..4bf8f4e3a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -2,16 +2,18 @@ This module contains class to define a RPC communications """ -from decimal import Decimal from datetime import datetime, timedelta +from decimal import Decimal + import arrow -from pandas import DataFrame import sqlalchemy as sql +from pandas import DataFrame + +from freqtrade import exchange from freqtrade.logger import Logger +from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State -from freqtrade import exchange -from freqtrade.misc import shorten_date class RPC(object): diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0d4e4403b..fce7a81f9 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,10 +5,12 @@ This module manage Telegram communication """ from typing import Any, Callable + from tabulate import tabulate from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater + from freqtrade.__init__ import __version__ from freqtrade.rpc.rpc import RPC diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 2247ecf27..ea37735b7 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -2,9 +2,10 @@ import talib.abstract as ta from pandas import DataFrame + import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.strategy.interface import IStrategy from freqtrade.indicator_helpers import fishers_inverse +from freqtrade.strategy.interface import IStrategy class_name = 'DefaultStrategy' diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dc9f33244..4eb73fb2e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -4,6 +4,7 @@ This module defines the interface to apply for strategies """ from abc import ABC, abstractmethod + from pandas import DataFrame diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index ed1dee10e..17bdb2c1f 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -7,11 +7,12 @@ import importlib import os import sys from collections import OrderedDict -from pandas import DataFrame -from freqtrade.logger import Logger -from freqtrade.constants import Constants -from freqtrade.strategy.interface import IStrategy +from pandas import DataFrame + +from freqtrade.constants import Constants +from freqtrade.logger import Logger +from freqtrade.strategy.interface import IStrategy sys.path.insert(0, r'../../user_data/strategies') diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index cb726633b..07dc45a3e 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -1,14 +1,15 @@ # pragma pylint: disable=missing-docstring -from datetime import datetime -from unittest.mock import MagicMock -from functools import reduce -import logging import json +import logging +from datetime import datetime +from functools import reduce +from unittest.mock import MagicMock + import arrow import pytest from jsonschema import validate -from telegram import Chat, Message, Update from sqlalchemy import create_engine +from telegram import Chat, Message, Update from freqtrade.analyze import Analyze from freqtrade.constants import Constants diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4f6eab083..f2874f2da 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1,15 +1,16 @@ # pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement # pragma pylint: disable=protected-access -from unittest.mock import MagicMock -from random import randint import logging -from requests.exceptions import RequestException -import pytest +from random import randint +from unittest.mock import MagicMock +import pytest +from requests.exceptions import RequestException + +import freqtrade.exchange as exchange from freqtrade import OperationalException from freqtrade.exchange import init, validate_pairs, buy, sell, get_balance, get_balances, \ get_ticker, get_ticker_history, cancel_order, get_name, get_fee -import freqtrade.exchange as exchange from freqtrade.tests.conftest import log_has API_INIT = False diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 058c25de1..2c66215c2 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -1,10 +1,12 @@ # pragma pylint: disable=missing-docstring, C0103, protected-access, unused-argument from unittest.mock import MagicMock + import pytest from requests.exceptions import ContentDecodingError -from freqtrade.exchange.bittrex import Bittrex + import freqtrade.exchange.bittrex as btx +from freqtrade.exchange.bittrex import Bittrex # Eat this flake8 diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 731b93176..021474d5c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,20 +1,21 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import json -import random import math -from typing import List +import random from copy import deepcopy +from typing import List from unittest.mock import MagicMock -from arrow import Arrow -import pandas as pd -import numpy as np -from freqtrade import optimize -from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration -from freqtrade.arguments import Arguments -from freqtrade.analyze import Analyze -from freqtrade.tests.conftest import default_conf, log_has +import numpy as np +import pandas as pd +from arrow import Arrow + +from freqtrade import optimize +from freqtrade.analyze import Analyze +from freqtrade.arguments import Arguments +from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration +from freqtrade.tests.conftest import default_conf, log_has # Avoid to reinit the same object again and again _BACKTESTING = Backtesting(default_conf()) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f29694c2f..6d376471a 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -3,6 +3,7 @@ import json import os from copy import deepcopy from unittest.mock import MagicMock + import pandas as pd from freqtrade.optimize.__init__ import load_tickerdata_file diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 43f46291f..e26d30534 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -1,13 +1,14 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 -import os import json +import os import uuid from shutil import copyfile + from freqtrade import optimize -from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ - download_backtesting_testdata, load_tickerdata_file, trim_tickerlist from freqtrade.misc import file_dump_json +from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ + download_backtesting_testdata, load_tickerdata_file, trim_tickerlist from freqtrade.tests.conftest import log_has # Change this if modifying BTC_UNITEST testdatafile diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 69ddc5ef5..4796b077e 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -6,23 +6,23 @@ Unit test file for rpc/telegram.py """ import re +from copy import deepcopy from datetime import datetime from random import randint from unittest.mock import MagicMock -from copy import deepcopy from sqlalchemy import create_engine from telegram import Update, Message, Chat from telegram.error import NetworkError from freqtrade import __version__ -from freqtrade.rpc.telegram import authorized_only from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.rpc.telegram import Telegram from freqtrade.persistence import Trade +from freqtrade.rpc.telegram import Telegram +from freqtrade.rpc.telegram import authorized_only from freqtrade.state import State -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap from freqtrade.tests.conftest import get_patched_freqtradebot, log_has +from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap class DummyCls(Telegram): diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 02b5630fc..2b91fbec5 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -1,8 +1,10 @@ import json + import pytest from pandas import DataFrame -from freqtrade.strategy.default_strategy import DefaultStrategy, class_name + from freqtrade.analyze import Analyze +from freqtrade.strategy.default_strategy import DefaultStrategy, class_name @pytest.fixture diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 77250659f..7ce9ae0ef 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging + from freqtrade.strategy.strategy import Strategy diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 4d1c31de2..558ea7ee5 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -5,8 +5,9 @@ Unit test file for analyse.py """ import datetime -from unittest.mock import MagicMock import logging +from unittest.mock import MagicMock + import arrow from pandas import DataFrame @@ -14,7 +15,6 @@ from freqtrade.analyze import Analyze, SignalType from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import log_has - # Avoid to reinit the same object again and again _ANALYZE = Analyze({'strategy': 'default_strategy'}) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 6ad507e85..3e0639304 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -6,6 +6,7 @@ Unit test file for arguments.py import argparse import logging + import pytest from freqtrade.arguments import Arguments diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 4ca92c343..002eac722 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -4,10 +4,10 @@ Unit test file for configuration.py """ import json - from copy import deepcopy -import pytest from unittest.mock import MagicMock + +import pytest from jsonschema import ValidationError from freqtrade.arguments import Arguments diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 86944e4f3..1f69a7d32 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -2,8 +2,8 @@ import pandas -from freqtrade.optimize import load_data from freqtrade.analyze import Analyze +from freqtrade.optimize import load_data from freqtrade.strategy.strategy import Strategy _pairs = ['BTC_ETH'] diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5d9b9aba1..d58b428da 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -5,22 +5,23 @@ Unit test file for freqtradebot.py """ import logging +import re import time -from unittest.mock import MagicMock from copy import deepcopy from typing import Dict, Optional +from unittest.mock import MagicMock + import arrow import pytest import requests -import re from sqlalchemy import create_engine -from freqtrade.tests.conftest import log_has from freqtrade import DependencyException, OperationalException from freqtrade.exchange import Exchanges from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.state import State from freqtrade.persistence import Trade +from freqtrade.state import State +from freqtrade.tests.conftest import log_has # Functions for recurrent object patching diff --git a/freqtrade/tests/test_indicator_helpers.py b/freqtrade/tests/test_indicator_helpers.py index 90330a6ef..87b085a0b 100644 --- a/freqtrade/tests/test_indicator_helpers.py +++ b/freqtrade/tests/test_indicator_helpers.py @@ -1,4 +1,5 @@ import pandas as pd + from freqtrade.indicator_helpers import went_up, went_down diff --git a/freqtrade/tests/test_logger.py b/freqtrade/tests/test_logger.py index d6e7a2e06..8e094b2d1 100644 --- a/freqtrade/tests/test_logger.py +++ b/freqtrade/tests/test_logger.py @@ -3,6 +3,7 @@ Unit test file for logger.py """ import logging + from freqtrade.logger import Logger diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 83da04495..71b816968 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -4,6 +4,7 @@ Unit test file for main.py import logging from unittest.mock import MagicMock + import pytest from freqtrade.main import main, set_loggers diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 1d2417332..3560b2db1 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -6,10 +6,11 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock + from freqtrade.analyze import Analyze -from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, common_datearray, file_dump_json) +from freqtrade.optimize.__init__ import load_tickerdata_file def test_shorten_date() -> None: diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index f6c5318ce..70199b12a 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,7 +1,9 @@ # pragma pylint: disable=missing-docstring, C0103 import os + import pytest from sqlalchemy import create_engine + from freqtrade.exchange import Exchanges from freqtrade.persistence import Trade, init, clean_dry_run_db diff --git a/freqtrade/tests/testdata/download_backtest_data.py b/freqtrade/tests/testdata/download_backtest_data.py index 0cb545b3a..ceb8388a1 100755 --- a/freqtrade/tests/testdata/download_backtest_data.py +++ b/freqtrade/tests/testdata/download_backtest_data.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """This script generate json data from bittrex""" -import sys import json +import sys from freqtrade import exchange -from freqtrade.exchange import Bittrex from freqtrade import misc +from freqtrade.exchange import Bittrex parser = misc.common_args_parser('download utility') parser.add_argument( From 7078bc00bd434ba77cbff2f214e40f2c47da0ca2 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 17 Mar 2018 23:30:31 +0100 Subject: [PATCH 51/56] rpc: apply correct typehints; remove redundant parentheses --- freqtrade/rpc/rpc.py | 68 +++++++++++++++++++----------------- freqtrade/rpc/rpc_manager.py | 2 +- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 4bf8f4e3a..2d7af1b50 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -4,6 +4,7 @@ This module contains class to define a RPC communications from datetime import datetime, timedelta from decimal import Decimal +from typing import Tuple, Any import arrow import sqlalchemy as sql @@ -32,7 +33,7 @@ class RPC(object): level=self.freqtrade.config.get('loglevel') ).get_logger() - def rpc_trade_status(self) -> (bool, Trade): + def rpc_trade_status(self) -> Tuple[bool, Any]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function @@ -41,9 +42,9 @@ class RPC(object): # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self.freqtrade.get_state() != State.RUNNING: - return (True, '*Status:* `trader is not running`') + return True, '*Status:* `trader is not running`' elif not trades: - return (True, '*Status:* `no active trade`') + return True, '*Status:* `no active trade`' else: result = [] for trade in trades: @@ -82,14 +83,14 @@ class RPC(object): ) if order else None, ) result.append(message) - return (False, result) + return False, result - def rpc_status_table(self) -> (bool, DataFrame): + def rpc_status_table(self) -> Tuple[bool, Any]: trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self.freqtrade.get_state() != State.RUNNING: - return (True, '*Status:* `trader is not running`') + return True, '*Status:* `trader is not running`' elif not trades: - return (True, '*Status:* `no active order`') + return True, '*Status:* `no active order`' else: trades_list = [] for trade in trades: @@ -109,14 +110,16 @@ class RPC(object): # consisting of (error_occured?, result) # Another approach would be to just return the # result, or raise error - return (False, df_statuses) + return False, df_statuses - def rpc_daily_profit(self, timescale, stake_currency, fiat_display_currency): + def rpc_daily_profit( + self, timescale: int, + stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: today = datetime.utcnow().date() profit_days = {} if not (isinstance(timescale, int) and timescale > 0): - return (True, '*Daily [n]:* `must be an integer greater than 0`') + return True, '*Daily [n]:* `must be an integer greater than 0`' fiat = self.freqtrade.fiat_converter for day in range(0, timescale): @@ -155,9 +158,10 @@ class RPC(object): ] for key, value in profit_days.items() ] - return (False, stats) + return False, stats - def rpc_trade_statistics(self, stake_currency, fiat_display_currency) -> None: + def rpc_trade_statistics( + self, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: """ :return: cumulative profit statistics. """ @@ -200,7 +204,7 @@ class RPC(object): .order_by(sql.text('profit_sum DESC')).first() if not best_pair: - return (True, '*Status:* `no closed trade`') + return True, '*Status:* `no closed trade`' bp_pair, bp_rate = best_pair @@ -241,7 +245,7 @@ class RPC(object): } ) - def rpc_balance(self, fiat_display_currency): + def rpc_balance(self, fiat_display_currency: str) -> Tuple[bool, Any]: """ :return: current account balance per crypto """ @@ -250,7 +254,7 @@ class RPC(object): if c['Balance'] or c['Available'] or c['Pending'] ] if not balances: - return (True, '`All balances are zero.`') + return True, '`All balances are zero.`' output = [] total = 0.0 @@ -277,17 +281,17 @@ class RPC(object): fiat = self.freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) - return (False, (output, total, symbol, value)) + return False, (output, total, symbol, value) def rpc_start(self) -> (bool, str): """ Handler for start. """ if self.freqtrade.get_state() == State.RUNNING: - return (True, '*Status:* `already running`') + return True, '*Status:* `already running`' self.freqtrade.update_state(State.RUNNING) - return (False, '`Starting trader ...`') + return False, '`Starting trader ...`' def rpc_stop(self) -> (bool, str): """ @@ -295,18 +299,18 @@ class RPC(object): """ if self.freqtrade.get_state() == State.RUNNING: self.freqtrade.update_state(State.STOPPED) - return (False, '`Stopping trader ...`') + return False, '`Stopping trader ...`' - return (True, '*Status:* `already stopped`') + return True, '*Status:* `already stopped`' # FIX: no test for this!!!! - def rpc_forcesell(self, trade_id) -> None: + def rpc_forcesell(self, trade_id) -> Tuple[bool, Any]: """ Handler for forcesell . Sells the given trade at current price :return: error or None """ - def _exec_forcesell(trade: Trade) -> str: + def _exec_forcesell(trade: Trade) -> None: # Check if there is there is an open order if trade.open_order_id: order = exchange.get_order(trade.open_order_id) @@ -328,13 +332,13 @@ class RPC(object): # ---- EOF def _exec_forcesell ---- if self.freqtrade.get_state() != State.RUNNING: - return (True, '`trader is not running`') + return True, '`trader is not running`' if trade_id == 'all': # Execute sell for all open orders for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): _exec_forcesell(trade) - return (False, '') + return False, '' # Query for trade trade = Trade.query.filter( @@ -345,18 +349,18 @@ class RPC(object): ).first() if not trade: self.logger.warning('forcesell: Invalid argument received') - return (True, 'Invalid argument.') + return True, 'Invalid argument.' _exec_forcesell(trade) - return (False, '') + return False, '' - def rpc_performance(self) -> None: + def rpc_performance(self) -> Tuple[bool, Any]: """ Handler for performance. Shows a performance statistic from finished trades """ if self.freqtrade.get_state() != State.RUNNING: - return (True, '`trader is not running`') + return True, '`trader is not running`' pair_rates = Trade.session.query(Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum'), @@ -369,15 +373,15 @@ class RPC(object): for (pair, rate, count) in pair_rates: trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) - return (False, trades) + return False, trades - def rpc_count(self) -> None: + def rpc_count(self) -> Tuple[bool, Any]: """ Returns the number of trades running :return: None """ if self.freqtrade.get_state() != State.RUNNING: - return (True, '`trader is not running`') + return True, '`trader is not running`' trades = Trade.query.filter(Trade.is_open.is_(True)).all() - return (False, trades) + return False, trades diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index cd3a694c2..fb18a8d73 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -28,7 +28,7 @@ class RPCManager(object): self.telegram = None self._init() - def _init(self): + def _init(self) -> None: """ Init RPC modules :return: From 33ddc540cf7f3b10ec7ad0ef0e758d91e392802f Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 18 Mar 2018 00:01:22 +0100 Subject: [PATCH 52/56] don't shadow built-in name tuple --- freqtrade/strategy/strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 17bdb2c1f..d7a89d1de 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -60,7 +60,7 @@ class Strategy(object): # Minimal ROI designed for the strategy self.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.custom_strategy.minimal_roi.items()}.items(), - key=lambda tuple: tuple[0])) # sort after converting to number + key=lambda t: t[0])) # sort after converting to number # Optimal stoploss designed for the strategy self.stoploss = float(self.custom_strategy.stoploss) From f6df7df9bf43dd549e2e0c9ec20765bef686a463 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 18 Mar 2018 00:02:02 +0100 Subject: [PATCH 53/56] modify args typehints --- freqtrade/arguments.py | 4 ++-- freqtrade/main.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a73a1945f..c69135117 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -26,7 +26,7 @@ class Arguments(object): self.common_args_parser() self._build_subcommands() - def get_parsed_arg(self) -> List[str]: + def get_parsed_arg(self) -> argparse.Namespace: """ Return the list of arguments :return: List[str] List of arguments @@ -37,7 +37,7 @@ class Arguments(object): return self.parsed_arg - def parse_args(self) -> List[str]: + def parse_args(self) -> argparse.Namespace: """ Parses given arguments and returns an argparse Namespace instance. """ diff --git a/freqtrade/main.py b/freqtrade/main.py index 8deeb7da6..d2cfc6f9f 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -6,7 +6,7 @@ Read the documentation to know what cli arguments you need. import logging import sys -from typing import Dict +from typing import List from freqtrade import (__version__) from freqtrade.arguments import Arguments @@ -17,7 +17,7 @@ from freqtrade.logger import Logger logger = Logger(name='freqtrade').get_logger() -def main(sysargv: Dict) -> None: +def main(sysargv: List[str]) -> None: """ This function will initiate the bot and start the trading loop. :return: None From a5c62b5c1082f758438ca0dfd5addc0d7cb1eed5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 18 Mar 2018 00:27:57 +0100 Subject: [PATCH 54/56] rpc/rpc.py: fix indentation --- freqtrade/rpc/rpc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2d7af1b50..b4592f78a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -196,11 +196,9 @@ class RPC(object): profit_all_percent.append(profit_percent) best_pair = Trade.session.query( - Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum') - )\ - .filter(Trade.is_open.is_(False))\ - .group_by(Trade.pair)\ + Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum') + ).filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ .order_by(sql.text('profit_sum DESC')).first() if not best_pair: From bc554faffb2736ecc9aafa34827d089bbb726955 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 18 Mar 2018 02:46:18 +0100 Subject: [PATCH 55/56] plot_profit: add missing typehints and fix mutable argument issue --- scripts/plot_profit.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 68434f1fb..022bbf33c 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -13,7 +13,8 @@ Optional Cli parameters import sys import json -from typing import Dict +from argparse import Namespace +from typing import List, Optional import numpy as np from plotly import tools @@ -34,8 +35,11 @@ logger = Logger(name="Graph profits").get_logger() # data:: [ pair, profit-%, enter, exit, time, duration] # data:: ["BTC_ETH", 0.0023975, "1515598200", "1515602100", "2018-01-10 07:30:00+00:00", 65] -def make_profit_array(data, px, min_date, interval, filter_pairs=[]): +def make_profit_array( + data: List, px: int, min_date: int, + interval: int, filter_pairs: Optional[List] = None) -> np.ndarray: pg = np.zeros(px) + filter_pairs = filter_pairs or [] # Go through the trades # and make an total profit # array @@ -63,7 +67,7 @@ def make_profit_array(data, px, min_date, interval, filter_pairs=[]): return pg -def plot_profit(args) -> None: +def plot_profit(args: Namespace) -> None: """ Plots the total profit for all pairs. Note, the profit calculation isn't realistic. @@ -183,14 +187,14 @@ def plot_profit(args) -> None: plot(fig, filename='freqtrade-profit-plot.html') -def define_index(min_date, max_date, interval): +def define_index(min_date: int, max_date: int, interval: int) -> int: """ Return the index of a specific date """ return int((max_date - min_date) / (interval * 60)) -def plot_parse_args(args): +def plot_parse_args(args: List[str]) -> Namespace: """ Parse args passed to the script :param args: Cli arguments @@ -205,7 +209,7 @@ def plot_parse_args(args): return arguments.parse_args() -def main(sysargv: Dict) -> None: +def main(sysargv: List[str]) -> None: """ This function will initiate the bot and start the trading loop. :return: None From 3553686e502c4abadf39f4e3f9544dc1560ac3f5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 18 Mar 2018 02:46:48 +0100 Subject: [PATCH 56/56] plot_dataframe: set missing typehints --- scripts/plot_dataframe.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e7a52b2bf..285ba6d97 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -13,8 +13,9 @@ Optional Cli parameters """ import sys +from argparse import Namespace -from typing import Dict +from typing import List from plotly import tools from plotly.offline import plot @@ -30,10 +31,9 @@ import freqtrade.optimize as optimize logger = Logger(name="Graph dataframe").get_logger() -def plot_analyzed_dataframe(args) -> None: +def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe - :param pair: pair as str :return: None """ pair = args.pair.replace('-', '_') @@ -153,7 +153,7 @@ def plot_analyzed_dataframe(args) -> None: plot(fig, filename='freqtrade-plot.html') -def plot_parse_args(args): +def plot_parse_args(args: List[str]) -> Namespace: """ Parse args passed to the script :param args: Cli arguments @@ -168,7 +168,7 @@ def plot_parse_args(args): return arguments.parse_args() -def main(sysargv: Dict) -> None: +def main(sysargv: List[str]) -> None: """ This function will initiate the bot and start the trading loop. :return: None