From 6e16c1d80d5173bc855a20f0e142805c9df004fa Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Jul 2018 21:42:17 +0200 Subject: [PATCH 01/47] add webhook test file --- freqtrade/tests/rpc/test_rpc_webhook.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 freqtrade/tests/rpc/test_rpc_webhook.py diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py new file mode 100644 index 000000000..e579c539a --- /dev/null +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -0,0 +1,16 @@ + + +def test__init__(): + # TODO: Implement me + pass + + +def test_send_msg(): + # TODO: Implement me + pass + + +def test__send_msg(): + """Test internal method - calling the actual api""" + # TODO: Implement me + pass From ae22af1ea3c0f3822bcd7e5463badb72f1e54171 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jul 2018 21:32:53 +0200 Subject: [PATCH 02/47] fix typo --- freqtrade/rpc/webhook.py | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 freqtrade/rpc/webhook.py diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py new file mode 100644 index 000000000..6a1abfe36 --- /dev/null +++ b/freqtrade/rpc/webhook.py @@ -0,0 +1,70 @@ +""" +This module manages webhook communication +""" +import logging +from typing import Any, Dict + +from requests import post, RequestException + +from freqtrade.rpc import RPC, RPCMessageType + + +logger = logging.getLogger(__name__) + +logger.debug('Included module rpc.webhook ...') + + +class Webhook(RPC): + """ This class handles all webhook communication """ + + def __init__(self, freqtrade) -> None: + """ + Init the Webhook class, and init the super class RPC + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + super().__init__(freqtrade) + + self._config = freqtrade.config + + def cleanup(self) -> None: + """ + Cleanup pending module resources. + This will do nothing for webhooks, they will simply not be called anymore + """ + pass + + def send_msg(self, msg: Dict[str, Any]) -> None: + """ Send a message to telegram channel """ + try: + + if msg['type'] == RPCMessageType.BUY_NOTIFICATION: + valuedict = self._config['webhook'].get('webhookbuy', None) + elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: + valuedict = self._config['webhook'].get('webhooksell', None) + elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: + valuedict = self._config['webhook'].get('webhookstatus', None) + else: + raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) + if not valuedict: + logger.info("Message type %s not configured for webhooks", msg['type']) + return + + payload = {"value1": valuedict.get('value1', '').format(**msg), + "value2": valuedict.get('value2', '').format(**msg), + "value3": valuedict.get('value3', '').format(**msg) + } + + self._send_msg(payload) + except KeyError as exc: + logger.exception("Problem calling Webhook. Please check your webhook configuration. " + "Exception: %s", exc) + + def _send_msg(self, payload: dict) -> None: + """does the actual call to the webhook""" + + try: + url = self._config['webhook']['url'] + post(url, data=payload) + except RequestException as exc: + logger.warning("Could not call webhook url. Exception: %s", exc) From fa8512789f331d5e15222d58daa13f974b6254ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jul 2018 21:33:01 +0200 Subject: [PATCH 03/47] add tests for webhook --- freqtrade/tests/rpc/test_rpc_webhook.py | 178 ++++++++++++++++++++++-- 1 file changed, 170 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py index e579c539a..99876269c 100644 --- a/freqtrade/tests/rpc/test_rpc_webhook.py +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -1,16 +1,178 @@ +from unittest.mock import MagicMock + +import pytest +from requests import RequestException -def test__init__(): - # TODO: Implement me +from freqtrade.rpc import RPCMessageType +from freqtrade.rpc.webhook import Webhook +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has + + +def get_webhook_dict() -> dict: + return { + "enabled": True, + "url": "https://maker.ifttt.com/trigger/freqtrade_test/with/key/c764udvJ5jfSlswVRukZZ2/", + "webhookbuy": { + "value1": "Buying {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhooksell": { + "value1": "Selling {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency}" + }, + "webhookstatus": { + "value1": "Status: {status}", + "value2": "", + "value3": "" + } + } + + +def test__init__(mocker, default_conf): + """ + Test __init__() method + """ + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + assert webhook._config == default_conf + + +def test_cleanup(default_conf, mocker) -> None: + """ + Test cleanup() method - not needed for webhook + """ pass -def test_send_msg(): - # TODO: Implement me - pass +def test_send_msg(default_conf, mocker): + """ Test send_msg for Webhook rpc class""" + default_conf["webhook"] = get_webhook_dict() + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + msg = { + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'market_url': "http://mockedurl/ETH_BTC", + 'limit': 0.005, + 'stake_amount': 0.8, + 'stake_amount_fiat': 500, + 'stake_currency': 'BTC', + 'fiat_currency': 'EUR' + } + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhookbuy"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhookbuy"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhookbuy"]["value3"].format(**msg)) + # Test sell + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': "profit", + 'market_url': "http://mockedurl/ETH_BTC", + 'limit': 0.005, + 'amount': 0.8, + 'open_rate': 0.004, + 'current_rate': 0.005, + 'profit_amount': 0.001, + 'profit_percent': 0.20, + 'stake_currency': 'BTC', + } + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhooksell"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhooksell"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhooksell"]["value3"].format(**msg)) + + # Test notification + msg = { + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': 'Unfilled sell order for BTC cancelled due to timeout' + } + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook.send_msg(msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhookstatus"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhookstatus"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhookstatus"]["value3"].format(**msg)) -def test__send_msg(): +def test_exception_send_msg(default_conf, mocker, caplog): + """Test misconfigured notification""" + default_conf["webhook"] = get_webhook_dict() + default_conf["webhook"]["webhookbuy"] = None + + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + webhook.send_msg({'type': RPCMessageType.BUY_NOTIFICATION}) + assert log_has(f"Message type {RPCMessageType.BUY_NOTIFICATION} not configured for webhooks", + caplog.record_tuples) + + default_conf["webhook"] = get_webhook_dict() + default_conf["webhook"]["webhookbuy"]["value1"] = "{DEADBEEF:8f}" + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + msg = { + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'market_url': "http://mockedurl/ETH_BTC", + 'limit': 0.005, + 'stake_amount': 0.8, + 'stake_amount_fiat': 500, + 'stake_currency': 'BTC', + 'fiat_currency': 'EUR' + } + webhook.send_msg(msg) + assert log_has("Problem calling Webhook. Please check your webhook configuration. " + "Exception: 'DEADBEEF'", caplog.record_tuples) + + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': 'DEADBEEF', + 'status': 'whatever' + } + with pytest.raises(NotImplementedError): + webhook.send_msg(msg) + + +def test__send_msg(default_conf, mocker, caplog): """Test internal method - calling the actual api""" - # TODO: Implement me - pass + + default_conf["webhook"] = get_webhook_dict() + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + msg = {'value1': 'DEADBEEF', + 'value2': 'ALIVEBEEF', + 'value3': 'FREQTRADE'} + post = MagicMock() + mocker.patch("freqtrade.rpc.webhook.post", post) + webhook._send_msg(msg) + + assert post.call_count == 1 + assert post.call_args[1] == {'data': msg} + assert post.call_args[0] == (default_conf['webhook']['url'], ) + + post = MagicMock(side_effect=RequestException) + mocker.patch("freqtrade.rpc.webhook.post", post) + webhook._send_msg(msg) + assert log_has('Could not call webhook url. Exception: ', caplog.record_tuples) From 25250f7c105d21e1ccd5a45fff720ba2e4dd5aa0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 13:38:06 +0200 Subject: [PATCH 04/47] don't hardcode post parameters --- freqtrade/rpc/webhook.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 6a1abfe36..35631182b 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -49,11 +49,9 @@ class Webhook(RPC): if not valuedict: logger.info("Message type %s not configured for webhooks", msg['type']) return - - payload = {"value1": valuedict.get('value1', '').format(**msg), - "value2": valuedict.get('value2', '').format(**msg), - "value3": valuedict.get('value3', '').format(**msg) - } + payload = {} + for k in valuedict: + payload[k] = valuedict[k].format(**msg) self._send_msg(payload) except KeyError as exc: From a4643066a85b1f526fe1c5820784b5aff1aa4543 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 13:39:21 +0200 Subject: [PATCH 05/47] allow more flexibility in webhook --- freqtrade/constants.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ec7765455..311f2fa72 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -100,6 +100,16 @@ CONF_SCHEMA = { }, 'required': ['enabled', 'token', 'chat_id'] }, + 'webhook': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'webhookbuy': {'type': 'object'}, + 'webhooksell': {'type': 'object'}, + 'webhookstatus': {'type': 'object'}, + 'chat_id': {'type': 'string'}, + }, + }, 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'internals': { From 71df41c4ebbdfe6f0321171132c82241641fc005 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:10:15 +0200 Subject: [PATCH 06/47] add documentation for rpc_webhook --- docs/configuration.md | 5 +++++ docs/index.md | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index dd16ef6b5..ddfa9834e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -41,6 +41,11 @@ The table below will list all configuration parameters. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. +| `webhook.enabled` | false | No | Enable useage of Webhook notifications +| `webhook.url` | false | No | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. +| `webhook.webhookbuy` | false | No | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. +| `webhook.webhooksell` | false | No | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. +| `webhook.webhookstatus` | false | No | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. | `db_url` | `sqlite:///tradesv3.sqlite` | No | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `True`. | `initial_state` | running | No | Defines the initial application state. More information below. | `strategy` | DefaultStrategy | No | Defines Strategy class to use. diff --git a/docs/index.md b/docs/index.md index fd6bf4378..f76bb125d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,6 +27,7 @@ Pull-request. Do not hesitate to reach us on - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) +- [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) - [Contribute to the project](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) - [How to contribute](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) - [Run tests & Check PEP8 compliance](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) From f55df7ba636f719a4600fe95962b662bc40721a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:11:52 +0200 Subject: [PATCH 07/47] improve README.md formatting (styling only) --- README.md | 74 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 929d40292..c1705ff41 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) - -Simple High frequency trading bot for crypto currencies designed to -support multi exchanges and be controlled via Telegram. +Simple High frequency trading bot for crypto currencies designed to support multi exchanges and be controlled via Telegram. ![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade-screenshot.png) ## Disclaimer + This software is for educational purposes only. Do not risk money which you are afraid to lose. USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. @@ -23,18 +22,18 @@ We strongly recommend you to have coding and Python knowledge. Do not hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported + - [X] [Bittrex](https://bittrex.com/) - [X] [Binance](https://www.binance.com/) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Features -- [x] **Based on Python 3.6+**: For botting on any operating system - -Windows, macOS and Linux + +- [x] **Based on Python 3.6+**: For botting on any operating system - Windows, macOS and Linux - [x] **Persistence**: Persistence is achieved through sqlite - [x] **Dry-run**: Run the bot without playing money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. -- [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell -strategy parameters with real exchange data. +- [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. - [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade. - [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. - [x] **Manageable via Telegram**: Manage the bot with Telegram @@ -43,38 +42,43 @@ strategy parameters with real exchange data. - [x] **Performance status report**: Provide a performance status of your current trades. ## Table of Contents + - [Quick start](#quick-start) - [Documentations](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) - - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) - - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) + - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) + - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) + - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) + - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Basic Usage](#basic-usage) - [Bot commands](#bot-commands) - [Telegram RPC commands](#telegram-rpc-commands) - [Support](#support) - - [Help](#help--slack) - - [Bugs](#bugs--issues) - - [Feature Requests](#feature-requests) - - [Pull Requests](#pull-requests) + - [Help](#help--slack) + - [Bugs](#bugs--issues) + - [Feature Requests](#feature-requests) + - [Pull Requests](#pull-requests) - [Requirements](#requirements) - - [Min hardware required](#min-hardware-required) - - [Software requirements](#software-requirements) + - [Min hardware required](#min-hardware-required) + - [Software requirements](#software-requirements) ## Quick start + Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. + ```bash git clone git@github.com:freqtrade/freqtrade.git git checkout develop cd freqtrade ./setup.sh --install ``` + _Windows installation is explained in [Installation doc](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md)_ - ## Documentation + We invite you to read the bot documentation to ensure you understand how the bot is working. + - [Index](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) @@ -86,7 +90,6 @@ We invite you to read the bot documentation to ensure you understand how the bot - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - ## Basic Usage ### Bot commands @@ -125,17 +128,15 @@ optional arguments: ``` ### Telegram RPC commands -Telegram is not mandatory. However, this is a great way to control your -bot. More details on our -[documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) + +Telegram is not mandatory. However, this is a great way to control your bot. More details on our [documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - `/start`: Starts the trader - `/stop`: Stops the trader - `/status [table]`: Lists all open trades - `/count`: Displays number of open trades - `/profit`: Lists cumulative profit from all finished trades -- `/forcesell |all`: Instantly sells the given trade -(Ignoring `minimum_roi`). +- `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair - `/balance`: Show account balance per currency - `/daily `: Shows profit or loss per day, over the last n days @@ -144,20 +145,23 @@ bot. More details on our ## Development branches -The project is currently setup in two main branches: -- `develop` - This branch has often new features, but might also cause -breaking changes. -- `master` - This branch contains the latest stable release. The bot -'should' be stable on this branch, and is generally well tested. +The project is currently setup in two main branches: + +- `develop` - This branch has often new features, but might also cause breaking changes. +- `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. ## Support + ### Help / Slack + For any questions not covered by the documentation or for further information about the bot, we encourage you to join our slack channel. + - [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) + If you discover a bug in the bot, please [search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) first. If it hasn't been reported, please @@ -166,6 +170,7 @@ ensure you follow the template guide so that our team can assist you as quickly as possible. ### [Feature Requests](https://github.com/freqtrade/freqtrade/labels/enhancement) + Have you a great idea to improve the bot you want to share? Please, first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement). If it hasn't been requested, please @@ -174,6 +179,7 @@ and ensure you follow the template guide so that it does not get lost in the bug reports. ### [Pull Requests](https://github.com/freqtrade/freqtrade/pulls) + Feel like our bot is missing a feature? We welcome your pull requests! Please read our [Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) @@ -181,16 +187,18 @@ to understand the requirements before sending your pull-requests. **Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. -**Important:** Always create your PR against the `develop` branch, not -`master`. +**Important:** Always create your PR against the `develop` branch, not `master`. ## Requirements ### Min hardware required + To run this bot we recommend you a cloud instance with a minimum of: -* Minimal (advised) system requirements: 2GB RAM, 1GB disk space, 2vCPU + +- Minimal (advised) system requirements: 2GB RAM, 1GB disk space, 2vCPU ### Software requirements + - [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) From 3ca161f1968ff0845e02f5880e09c214301257d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:13:49 +0200 Subject: [PATCH 08/47] Add webhook config --- docs/webhook-config.md | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 docs/webhook-config.md diff --git a/docs/webhook-config.md b/docs/webhook-config.md new file mode 100644 index 000000000..71524187a --- /dev/null +++ b/docs/webhook-config.md @@ -0,0 +1,74 @@ +# Webhook usage + +This page explains how to configure your bot to talk to webhooks. + +## Configuration + +Enable webhooks by adding a webhook-section to your configuration file, and setting `webhook.enabled` to `true`. + +Sample configuration (tested using IFTTT). + +```json + "webhook": { + "enabled": true, + "url": "https://maker.ifttt.com/trigger//with/key//", + "webhookbuy": { + "value1": "Buying {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhooksell": { + "value1": "Selling {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency}" + }, + "webhookstatus": { + "value1": "Status: {status}", + "value2": "", + "value3": "" + } + }, +``` + +The url in `webhook.url` should point to the correct url for your webhook. If you're using [IFTTT](https://ifttt.com) (as shown in the sample above) please insert our event and key to the url. + +Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called. + +### Webhookbuy + +The fields in `webhook.webhookbuy` are filled when the bot executes a buy. Parameters are filled using string.format. +Possible parameters are: + +* exchange +* pair +* market_url +* limit +* stake_amount +* stake_amount_fiat +* stake_currency +* fiat_currency + +### Webhooksell + +The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format. +Possible parameters are: + +* exchange +* pair +* gain +* market_url +* limit +* amount +* open_rate +* current_rate +* profit_amount +* profit_percent +* profit_fiat +* stake_currency +* fiat_currency + +### Webhookstatus + +The fields in `webhook.webhookstatus` are used for regular status messages (Started / Stopped / ...). Parameters are filled using string.format. + +The only possible value here is `{status}`. From 144d308e5ed55d10103453af85108eee863d4687 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 19:59:17 +0200 Subject: [PATCH 09/47] Allow enabling of webhook --- freqtrade/freqtradebot.py | 3 +-- freqtrade/rpc/rpc_manager.py | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 72b5190b9..aed8f62dd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,8 +19,7 @@ from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade -from freqtrade.rpc import RPCMessageType -from freqtrade.rpc import RPCManager +from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 34094ee20..f05a1882f 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -23,6 +23,12 @@ class RPCManager(object): from freqtrade.rpc.telegram import Telegram self.registered_modules.append(Telegram(freqtrade)) + # Enable Webhook + if freqtrade.config.get('webhook', {}).get('enabled', False): + logger.info('Enabling rpc.webhook...') + from freqtrade.rpc.webhook import Webhook + self.registered_modules.append(Webhook(freqtrade)) + def cleanup(self) -> None: """ Stops all enabled rpc modules """ logger.info('Cleaning up rpc modules ...') From ee2f6ccbe95d2ec44d7b319d5cf697fa36d48409 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 20:17:45 +0200 Subject: [PATCH 10/47] Add test for enable_webhook --- freqtrade/rpc/rpc_manager.py | 2 +- freqtrade/tests/rpc/test_rpc_manager.py | 30 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index f05a1882f..022578378 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -25,7 +25,7 @@ class RPCManager(object): # Enable Webhook if freqtrade.config.get('webhook', {}).get('enabled', False): - logger.info('Enabling rpc.webhook...') + logger.info('Enabling rpc.webhook ...') from freqtrade.rpc.webhook import Webhook self.registered_modules.append(Webhook(freqtrade)) diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 1f9b034b9..667b3d21d 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -127,3 +127,33 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: assert log_has("Sending rpc message: {'type': status, 'status': 'test'}", caplog.record_tuples) assert telegram_mock.call_count == 1 + + +def test_init_webhook_disabled(mocker, default_conf, caplog) -> None: + """ Test _init() method with Webhook disabled """ + caplog.set_level(logging.DEBUG) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + conf['webhook'] = {'enabled': False} + + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) + + assert not log_has('Enabling rpc.webhook ...', caplog.record_tuples) + assert rpc_manager.registered_modules == [] + + +def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: + """ + Test _init() method with Webhook enabled + """ + caplog.set_level(logging.DEBUG) + default_conf['telegram']['enabled'] = False + default_conf['webhook'] = {'enabled': True} + + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) + + assert log_has('Enabling rpc.webhook ...', caplog.record_tuples) + len_modules = len(rpc_manager.registered_modules) + assert len_modules == 1 + assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] From 6336d8a0e27e41daf36df6a184486be6437ca25c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 21:43:52 +0200 Subject: [PATCH 11/47] remove copy leftover --- freqtrade/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 311f2fa72..385dac1d1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -107,7 +107,6 @@ CONF_SCHEMA = { 'webhookbuy': {'type': 'object'}, 'webhooksell': {'type': 'object'}, 'webhookstatus': {'type': 'object'}, - 'chat_id': {'type': 'string'}, }, }, 'db_url': {'type': 'string'}, From 120fc29643419ff25d5bc2ee86ea2509c5fec281 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 21:54:31 +0200 Subject: [PATCH 12/47] use dict comprehension --- freqtrade/rpc/webhook.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 35631182b..b3c22ee0e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -49,10 +49,8 @@ class Webhook(RPC): if not valuedict: logger.info("Message type %s not configured for webhooks", msg['type']) return - payload = {} - for k in valuedict: - payload[k] = valuedict[k].format(**msg) + payload = {key: value.format(**msg) for (key, value) in valuedict.items()} self._send_msg(payload) except KeyError as exc: logger.exception("Problem calling Webhook. Please check your webhook configuration. " From 12846272195037ee83ada4189b7ade4499446c21 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Jul 2018 13:29:34 +0200 Subject: [PATCH 13/47] move url to private class level --- freqtrade/rpc/webhook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index b3c22ee0e..bfc82b8d6 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -26,6 +26,7 @@ class Webhook(RPC): super().__init__(freqtrade) self._config = freqtrade.config + self._url = self._config['webhook']['url'] def cleanup(self) -> None: """ @@ -57,10 +58,9 @@ class Webhook(RPC): "Exception: %s", exc) def _send_msg(self, payload: dict) -> None: - """does the actual call to the webhook""" + """do the actual call to the webhook""" try: - url = self._config['webhook']['url'] - post(url, data=payload) + post(self._url, data=payload) except RequestException as exc: logger.warning("Could not call webhook url. Exception: %s", exc) From 278e7159bccc0f19d8becce00e5687f3e2b17183 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Jul 2018 13:29:50 +0200 Subject: [PATCH 14/47] adjust webhook tests --- freqtrade/tests/rpc/test_rpc_manager.py | 2 +- freqtrade/tests/rpc/test_rpc_webhook.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 667b3d21d..4686cf5ca 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -149,7 +149,7 @@ def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: """ caplog.set_level(logging.DEBUG) default_conf['telegram']['enabled'] = False - default_conf['webhook'] = {'enabled': True} + default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py index 99876269c..b9c005d73 100644 --- a/freqtrade/tests/rpc/test_rpc_webhook.py +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -35,6 +35,7 @@ def test__init__(mocker, default_conf): """ Test __init__() method """ + default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) assert webhook._config == default_conf From a74147c472b2930d910266133d1d74609c438afa Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 9 Jul 2018 19:27:36 +0300 Subject: [PATCH 15/47] move strategy initialization outside Analyze --- freqtrade/analyze.py | 6 +++--- freqtrade/freqtradebot.py | 12 +++++++----- freqtrade/optimize/backtesting.py | 4 +++- freqtrade/tests/optimize/test_backtesting.py | 3 ++- freqtrade/tests/test_analyze.py | 5 +++-- freqtrade/tests/test_dataframe.py | 12 ++++++------ freqtrade/tests/test_freqtradebot.py | 3 +-- freqtrade/tests/test_misc.py | 3 ++- 8 files changed, 27 insertions(+), 21 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 493228e68..71d96264f 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -12,7 +12,7 @@ from pandas import DataFrame, to_datetime from freqtrade import constants from freqtrade.exchange import Exchange from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.strategy.resolver import IStrategy logger = logging.getLogger(__name__) @@ -30,13 +30,13 @@ class Analyze(object): Analyze class contains everything the bot need to determine if the situation is good for buying or selling. """ - def __init__(self, config: dict) -> None: + def __init__(self, config: dict, strategy: IStrategy) -> None: """ Init Analyze :param config: Bot configuration (use the one from Configuration()) """ self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.strategy = strategy @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 72b5190b9..ad61b5533 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -22,6 +22,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.rpc import RPCManager from freqtrade.state import State +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -49,7 +50,8 @@ class FreqtradeBot(object): # Init objects self.config = config - self.analyze = Analyze(self.config) + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.analyze = Analyze(self.config, self.strategy) self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None @@ -293,8 +295,8 @@ class FreqtradeBot(object): return None amount_reserve_percent = 1 - 0.05 # reserve 5% + stoploss - if self.analyze.get_stoploss() is not None: - amount_reserve_percent += self.analyze.get_stoploss() + if self.strategy.stoploss is not None: + amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) return min(min_stake_amounts)/amount_reserve_percent @@ -305,7 +307,7 @@ class FreqtradeBot(object): if one pair triggers the buy_signal a new trade record gets created :return: True if a trade object has been created and persisted, False otherwise """ - interval = self.analyze.get_ticker_interval() + interval = self.strategy.ticker_interval stake_amount = self._get_trade_stake_amount() if not stake_amount: @@ -499,7 +501,7 @@ class FreqtradeBot(object): experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): (buy, sell) = self.analyze.get_signal(self.exchange, - trade.pair, self.analyze.get_ticker_interval()) + trade.pair, self.strategy.ticker_interval) if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..67d4bb2e9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -21,6 +21,7 @@ from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -52,7 +53,8 @@ class Backtesting(object): """ def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.analyze = Analyze(self.config) + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.analyze = Analyze(self.config, self.strategy) self.ticker_interval = self.analyze.strategy.ticker_interval self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe self.populate_buy_trend = self.analyze.populate_buy_trend diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6fbf71e40..89f5a0bb7 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -18,6 +18,7 @@ from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange +from freqtrade.strategy.default_strategy import DefaultStrategy def get_args(args) -> List[str]: @@ -348,7 +349,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: assert len(data['UNITTEST/BTC']) == 99 # Load Analyze to compare the result between Backtesting function and Analyze are the same - analyze = Analyze(default_conf) + analyze = Analyze(default_conf, DefaultStrategy()) data2 = analyze.tickerdata_to_dataframe(tickerlist) assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 6e035d842..dc7410ffc 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -14,9 +14,10 @@ from freqtrade.analyze import Analyze, SignalType from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.strategy.default_strategy import DefaultStrategy # Avoid to reinit the same object again and again -_ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) +_ANALYZE = Analyze({}, DefaultStrategy()) def test_signaltype_object() -> None: @@ -189,7 +190,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: """ Test Analyze.tickerdata_to_dataframe() method """ - analyze = Analyze(default_conf) + analyze = Analyze(default_conf, DefaultStrategy()) timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index fd461a503..f4b26eb1d 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -9,26 +9,26 @@ from freqtrade.strategy.resolver import StrategyResolver _pairs = ['ETH/BTC'] -def load_dataframe_pair(pairs): +def load_dataframe_pair(pairs, strategy): ld = load_data(None, ticker_interval='5m', pairs=pairs) assert isinstance(ld, dict) assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - analyze = Analyze({'strategy': 'DefaultStrategy'}) + analyze = Analyze({}, strategy) dataframe = analyze.analyze_ticker(dataframe) return dataframe def test_dataframe_load(): - StrategyResolver({'strategy': 'DefaultStrategy'}) - dataframe = load_dataframe_pair(_pairs) + strategy = StrategyResolver({'strategy': 'DefaultStrategy'}).strategy + dataframe = load_dataframe_pair(_pairs, strategy) assert isinstance(dataframe, pandas.core.frame.DataFrame) def test_dataframe_columns_exists(): - StrategyResolver({'strategy': 'DefaultStrategy'}) - dataframe = load_dataframe_pair(_pairs) + strategy = StrategyResolver({'strategy': 'DefaultStrategy'}).strategy + dataframe = load_dataframe_pair(_pairs, strategy) assert 'high' in dataframe.columns assert 'low' in dataframe.columns assert 'close' in dataframe.columns diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 450504f57..c628a9da3 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -316,9 +316,8 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) - mocker.patch('freqtrade.freqtradebot.Analyze.get_stoploss', MagicMock(return_value=-0.05)) freqtrade = FreqtradeBot(default_conf) - + freqtrade.strategy.stoploss = -0.05 # no pair found mocker.patch( 'freqtrade.exchange.Exchange.get_markets', diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index e2ba40dee..c30225132 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -11,6 +11,7 @@ from freqtrade.analyze import Analyze from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.strategy.default_strategy import DefaultStrategy def test_shorten_date() -> None: @@ -47,7 +48,7 @@ def test_common_datearray(default_conf) -> None: Test common_datearray() :return: None """ - analyze = Analyze(default_conf) + analyze = Analyze(default_conf, DefaultStrategy()) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = analyze.tickerdata_to_dataframe(tickerlist) From 85e6c9585ad8bed469f4a47b949036c1597a50a4 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 10 Jul 2018 09:11:36 +0300 Subject: [PATCH 16/47] remove pass-through methods from Analyze --- freqtrade/analyze.py | 48 +++---------------------------- freqtrade/optimize/backtesting.py | 4 +-- freqtrade/tests/test_analyze.py | 39 +------------------------ 3 files changed, 7 insertions(+), 84 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 71d96264f..c27a31bb6 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -64,46 +64,6 @@ class Analyze(object): frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - return self.strategy.populate_indicators(dataframe=dataframe) - - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - return self.strategy.populate_buy_trend(dataframe=dataframe) - - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - return self.strategy.populate_sell_trend(dataframe=dataframe) - - def get_ticker_interval(self) -> str: - """ - Return ticker interval to use - :return: Ticker interval value to use - """ - return self.strategy.ticker_interval - - def get_stoploss(self) -> float: - """ - Return stoploss to use - :return: Strategy stoploss value to use - """ - return self.strategy.stoploss - def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame @@ -111,9 +71,9 @@ class Analyze(object): :return DataFrame with ticker data and indicator data """ dataframe = self.parse_ticker_dataframe(ticker_history) - dataframe = self.populate_indicators(dataframe) - dataframe = self.populate_buy_trend(dataframe) - dataframe = self.populate_sell_trend(dataframe) + dataframe = self.strategy.populate_indicators(dataframe) + dataframe = self.strategy.populate_buy_trend(dataframe) + dataframe = self.strategy.populate_sell_trend(dataframe) return dataframe def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: @@ -267,5 +227,5 @@ class Analyze(object): """ Creates a dataframe and populates indicators for given ticker data """ - return {pair: self.populate_indicators(self.parse_ticker_dataframe(pair_data)) + return {pair: self.strategy.populate_indicators(self.parse_ticker_dataframe(pair_data)) for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 67d4bb2e9..254bf80c6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -57,8 +57,8 @@ class Backtesting(object): self.analyze = Analyze(self.config, self.strategy) self.ticker_interval = self.analyze.strategy.ticker_interval self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe - self.populate_buy_trend = self.analyze.populate_buy_trend - self.populate_sell_trend = self.analyze.populate_sell_trend + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend # Reset keys for backtesting self.config['exchange']['key'] = '' diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index dc7410ffc..4cd68dd86 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import arrow from pandas import DataFrame -from freqtrade.analyze import Analyze, SignalType +from freqtrade.analyze import Analyze from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -20,31 +20,6 @@ from freqtrade.strategy.default_strategy import DefaultStrategy _ANALYZE = Analyze({}, DefaultStrategy()) -def test_signaltype_object() -> None: - """ - Test the SignalType object has the mandatory Constants - :return: None - """ - assert hasattr(SignalType, 'BUY') - assert hasattr(SignalType, 'SELL') - - -def test_analyze_object() -> None: - """ - Test the Analyze object has the mandatory methods - :return: None - """ - assert hasattr(Analyze, 'parse_ticker_dataframe') - assert hasattr(Analyze, 'populate_indicators') - assert hasattr(Analyze, 'populate_buy_trend') - assert hasattr(Analyze, 'populate_sell_trend') - assert hasattr(Analyze, 'analyze_ticker') - assert hasattr(Analyze, 'get_signal') - assert hasattr(Analyze, 'should_sell') - assert hasattr(Analyze, 'min_roi_reached') - assert hasattr(Analyze, 'stop_loss_reached') - - def test_dataframe_correct_length(result): dataframe = Analyze.parse_ticker_dataframe(result) assert len(result.index) - 1 == len(dataframe.index) # last partial candle removed @@ -55,18 +30,6 @@ def test_dataframe_correct_columns(result): ['date', 'open', 'high', 'low', 'close', 'volume'] -def test_populates_buy_trend(result): - # Load the default strategy for the unit test, because this logic is done in main.py - dataframe = _ANALYZE.populate_buy_trend(_ANALYZE.populate_indicators(result)) - assert 'buy' in dataframe.columns - - -def test_populates_sell_trend(result): - # Load the default strategy for the unit test, because this logic is done in main.py - dataframe = _ANALYZE.populate_sell_trend(_ANALYZE.populate_indicators(result)) - assert 'sell' in dataframe.columns - - def test_returns_latest_buy_signal(mocker, default_conf): mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) exchange = get_patched_exchange(mocker, default_conf) From f6b8c2b40fc8c6df42cbdd45de8c26ca8b0ded8b Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 10 Jul 2018 13:04:37 +0300 Subject: [PATCH 17/47] move parse_ticker_dataframe outside Analyze class --- freqtrade/analyze.py | 56 +++++++++---------- freqtrade/tests/conftest.py | 4 +- .../tests/strategy/test_default_strategy.py | 4 +- freqtrade/tests/test_analyze.py | 6 +- freqtrade/tests/test_misc.py | 4 +- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index c27a31bb6..7a8ba3fb0 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -17,6 +17,32 @@ from freqtrade.strategy.resolver import IStrategy logger = logging.getLogger(__name__) +def parse_ticker_dataframe(ticker: list) -> DataFrame: + """ + Analyses the trend for the given ticker history + :param ticker: See exchange.get_ticker_history + :return: DataFrame + """ + cols = ['date', 'open', 'high', 'low', 'close', 'volume'] + frame = DataFrame(ticker, columns=cols) + + frame['date'] = to_datetime(frame['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + + # group by index and aggregate results to eliminate duplicate ticks + frame = frame.groupby(by='date', as_index=False, sort=True).agg({ + 'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'max', + }) + frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle + return frame + + class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -38,39 +64,13 @@ class Analyze(object): self.config = config self.strategy = strategy - @staticmethod - def parse_ticker_dataframe(ticker: list) -> DataFrame: - """ - Analyses the trend for the given ticker history - :param ticker: See exchange.get_ticker_history - :return: DataFrame - """ - cols = ['date', 'open', 'high', 'low', 'close', 'volume'] - frame = DataFrame(ticker, columns=cols) - - frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) - - # group by index and aggregate results to eliminate duplicate ticks - frame = frame.groupby(by='date', as_index=False, sort=True).agg({ - 'open': 'first', - 'high': 'max', - 'low': 'min', - 'close': 'last', - 'volume': 'max', - }) - frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle - return frame - def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - dataframe = self.parse_ticker_dataframe(ticker_history) + dataframe = parse_ticker_dataframe(ticker_history) dataframe = self.strategy.populate_indicators(dataframe) dataframe = self.strategy.populate_buy_trend(dataframe) dataframe = self.strategy.populate_sell_trend(dataframe) @@ -227,5 +227,5 @@ class Analyze(object): """ Creates a dataframe and populates indicators for given ticker data """ - return {pair: self.strategy.populate_indicators(self.parse_ticker_dataframe(pair_data)) + return {pair: self.strategy.populate_indicators(parse_ticker_dataframe(pair_data)) for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 9c86d1ece..078dba447 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,7 +12,7 @@ from jsonschema import validate from telegram import Chat, Message, Update from freqtrade import constants -from freqtrade.analyze import Analyze +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot @@ -616,7 +616,7 @@ def tickers(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file: - return Analyze.parse_ticker_dataframe(json.load(data_file)) + return parse_ticker_dataframe(json.load(data_file)) # FIX: # Create an fixture/function diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 900fc2234..2175dc9b3 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -3,14 +3,14 @@ import json import pytest from pandas import DataFrame -from freqtrade.analyze import Analyze +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.strategy.default_strategy import DefaultStrategy @pytest.fixture def result(): with open('freqtrade/tests/testdata/ETH_BTC-1m.json') as data_file: - return Analyze.parse_ticker_dataframe(json.load(data_file)) + return parse_ticker_dataframe(json.load(data_file)) def test_default_strategy_structure(): diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 4cd68dd86..042422c5a 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import arrow from pandas import DataFrame -from freqtrade.analyze import Analyze +from freqtrade.analyze import Analyze, parse_ticker_dataframe from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -21,7 +21,7 @@ _ANALYZE = Analyze({}, DefaultStrategy()) def test_dataframe_correct_length(result): - dataframe = Analyze.parse_ticker_dataframe(result) + dataframe = parse_ticker_dataframe(result) assert len(result.index) - 1 == len(dataframe.index) # last partial candle removed @@ -145,7 +145,7 @@ def test_parse_ticker_dataframe(ticker_history): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] # Test file with BV data - dataframe = Analyze.parse_ticker_dataframe(ticker_history) + dataframe = parse_ticker_dataframe(ticker_history) assert dataframe.columns.tolist() == columns diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index c30225132..15ed1550b 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -7,7 +7,7 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock -from freqtrade.analyze import Analyze +from freqtrade.analyze import Analyze, parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file @@ -29,7 +29,7 @@ def test_datesarray_to_datetimearray(ticker_history): Test datesarray_to_datetimearray() function :return: None """ - dataframes = Analyze.parse_ticker_dataframe(ticker_history) + dataframes = parse_ticker_dataframe(ticker_history) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) From aeb4102bcbb1c0d27e3c3888c4cd0fa8005c725f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 16 Jul 2018 08:11:17 +0300 Subject: [PATCH 18/47] refactor Analyze class methods to base Strategy class --- freqtrade/analyze.py | 198 ---------------- freqtrade/freqtradebot.py | 11 +- freqtrade/optimize/backtesting.py | 10 +- freqtrade/optimize/hyperopt.py | 6 +- freqtrade/strategy/__init__.py | 4 +- freqtrade/strategy/interface.py | 190 ++++++++++++++- freqtrade/strategy/resolver.py | 12 +- freqtrade/tests/conftest.py | 4 +- freqtrade/tests/optimize/test_backtesting.py | 10 +- freqtrade/tests/rpc/test_rpc.py | 22 +- freqtrade/tests/rpc/test_rpc_telegram.py | 51 ++-- .../tests/strategy/test_default_strategy.py | 2 +- freqtrade/tests/strategy/test_strategy.py | 21 +- freqtrade/tests/test_analyze.py | 220 +++++++++--------- freqtrade/tests/test_dataframe.py | 4 +- freqtrade/tests/test_freqtradebot.py | 161 +++++++------ freqtrade/tests/test_misc.py | 6 +- 17 files changed, 473 insertions(+), 459 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 7a8ba3fb0..254c16309 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -2,18 +2,8 @@ Functions to analyze ticker data with indicators and produce buy and sell signals """ import logging -from datetime import datetime -from enum import Enum -from typing import Dict, List, Tuple - -import arrow from pandas import DataFrame, to_datetime -from freqtrade import constants -from freqtrade.exchange import Exchange -from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import IStrategy - logger = logging.getLogger(__name__) @@ -41,191 +31,3 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame - - -class SignalType(Enum): - """ - Enum to distinguish between buy and sell signals - """ - BUY = "buy" - SELL = "sell" - - -class Analyze(object): - """ - Analyze class contains everything the bot need to determine if the situation is good for - buying or selling. - """ - def __init__(self, config: dict, strategy: IStrategy) -> None: - """ - Init Analyze - :param config: Bot configuration (use the one from Configuration()) - """ - self.config = config - self.strategy = strategy - - def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: - """ - Parses the given ticker history and returns a populated DataFrame - add several TA indicators and buy signal to it - :return DataFrame with ticker data and indicator data - """ - dataframe = parse_ticker_dataframe(ticker_history) - dataframe = self.strategy.populate_indicators(dataframe) - dataframe = self.strategy.populate_buy_trend(dataframe) - dataframe = self.strategy.populate_sell_trend(dataframe) - return dataframe - - def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: - """ - Calculates current signal based several technical analysis indicators - :param pair: pair in format ANT/BTC - :param interval: Interval to use (in min) - :return: (Buy, Sell) A bool-tuple indicating buy/sell signal - """ - ticker_hist = exchange.get_ticker_history(pair, interval) - if not ticker_hist: - logger.warning('Empty ticker history for pair %s', pair) - return False, False - - try: - dataframe = self.analyze_ticker(ticker_hist) - except ValueError as error: - logger.warning( - 'Unable to analyze ticker for pair %s: %s', - pair, - str(error) - ) - return False, False - except Exception as error: - logger.exception( - 'Unexpected error when analyzing ticker for pair %s: %s', - pair, - str(error) - ) - return False, False - - if dataframe.empty: - logger.warning('Empty dataframe for pair %s', pair) - return False, False - - latest = dataframe.iloc[-1] - - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): - logger.warning( - 'Outdated history for pair %s. Last tick is %s minutes old', - pair, - (arrow.utcnow() - signal_date).seconds // 60 - ) - return False, False - - (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 - logger.debug( - 'trigger: %s (pair=%s) buy=%s sell=%s', - latest['date'], - pair, - str(buy), - str(sell) - ) - return buy, sell - - def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: - """ - This function evaluate if on the condition required to trigger a sell has been reached - if the threshold is reached and updates the trade record. - :return: True if trade should be sold, False otherwise - """ - current_profit = trade.calc_profit_percent(rate) - if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit): - return True - - experimental = self.config.get('experimental', {}) - - if buy and experimental.get('ignore_roi_if_buy_signal', False): - logger.debug('Buy signal still active - not selling.') - return False - - # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) - if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): - logger.debug('Required profit reached. Selling..') - return True - - if experimental.get('sell_profit_only', False): - logger.debug('Checking if trade is profitable..') - if trade.calc_profit(rate=rate) <= 0: - return False - if sell and not buy and experimental.get('use_sell_signal', False): - logger.debug('Sell signal received. Selling..') - return True - - return False - - def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> bool: - """ - Based on current profit of the trade and configured (trailing) stoploss, - decides to sell or not - """ - - trailing_stop = self.config.get('trailing_stop', False) - - trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) - - # evaluate if the stoploss was hit - if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: - - if trailing_stop: - logger.debug( - f"HIT STOP: current price at {current_rate:.6f}, " - f"stop loss is {trade.stop_loss:.6f}, " - f"initial stop loss was at {trade.initial_stop_loss:.6f}, " - f"trade opened at {trade.open_rate:.6f}") - logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") - - logger.debug('Stop loss hit.') - return True - - # update the stop loss afterwards, after all by definition it's supposed to be hanging - if trailing_stop: - - # check if we have a special stop loss for positive condition - # and if profit is positive - stop_loss_value = self.strategy.stoploss - if 'trailing_stop_positive' in self.config and current_profit > 0: - - # Ignore mypy error check in configuration that this is a float - stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore - logger.debug(f"using positive stop loss mode: {stop_loss_value} " - f"since we have profit {current_profit}") - - trade.adjust_stop_loss(current_rate, stop_loss_value) - - return False - - def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: - """ - Based an earlier trade and current price and ROI configuration, decides whether bot should - sell - :return True if bot should sell at current rate - """ - - # Check if time matches and current rate is above threshold - time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration, threshold in self.strategy.minimal_roi.items(): - if time_diff <= duration: - return False - if current_profit > threshold: - return True - - return False - - def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: - """ - Creates a dataframe and populates indicators for given ticker data - """ - return {pair: self.strategy.populate_indicators(parse_ticker_dataframe(pair_data)) - for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ad61b5533..0d916f570 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -15,7 +15,6 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) -from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade @@ -51,7 +50,7 @@ class FreqtradeBot(object): # Init objects self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.analyze = Analyze(self.config, self.strategy) +# self.analyze = Analyze(self.config, self.strategy) self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None @@ -330,7 +329,7 @@ class FreqtradeBot(object): # Pick pair based on buy signals for _pair in whitelist: - (buy, sell) = self.analyze.get_signal(self.exchange, _pair, interval) + (buy, sell) = self.strategy.get_signal(self.exchange, _pair, interval) if buy and not sell: return self.execute_buy(_pair, stake_amount) return False @@ -500,10 +499,10 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - (buy, sell) = self.analyze.get_signal(self.exchange, - trade.pair, self.strategy.ticker_interval) + (buy, sell) = self.strategy.get_signal(self.exchange, + trade.pair, self.strategy.ticker_interval) - if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): + if self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) return True logger.info('Found no sell signals for whitelisted currencies. Trying again..') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 254bf80c6..9c124f35b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,7 +15,6 @@ from tabulate import tabulate import freqtrade.optimize as optimize from freqtrade import DependencyException, constants -from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange @@ -54,9 +53,8 @@ class Backtesting(object): def __init__(self, config: Dict[str, Any]) -> None: self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.analyze = Analyze(self.config, self.strategy) - self.ticker_interval = self.analyze.strategy.ticker_interval - self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe + self.ticker_interval = self.strategy.ticker_interval + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe self.populate_buy_trend = self.strategy.populate_buy_trend self.populate_sell_trend = self.strategy.populate_sell_trend @@ -153,8 +151,8 @@ class Backtesting(object): trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 buy_signal = sell_row.buy - if self.analyze.should_sell(trade, sell_row.open, sell_row.date, buy_signal, - sell_row.sell): + if self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, + sell_row.sell): return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 72bf34eb3..3b4652883 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -267,13 +267,13 @@ class Hyperopt(Backtesting): params = self.get_args(_params) if self.has_space('roi'): - self.analyze.strategy.minimal_roi = self.generate_roi_table(params) + self.strategy.minimal_roi = self.generate_roi_table(params) if self.has_space('buy'): self.populate_buy_trend = self.buy_strategy_generator(params) if self.has_space('stoploss'): - self.analyze.strategy.stoploss = params['stoploss'] + self.strategy.stoploss = params['stoploss'] processed = load(TICKERDATA_PICKLE) results = self.backtest( @@ -351,7 +351,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.populate_indicators = Hyperopt.populate_indicators # type: ignore dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index e1dc7bb3f..283426dfa 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -7,7 +7,7 @@ from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) -def import_strategy(strategy: IStrategy) -> IStrategy: +def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: """ Imports given Strategy instance to global scope of freqtrade.strategy and returns an instance of it @@ -29,4 +29,4 @@ def import_strategy(strategy: IStrategy) -> IStrategy: # Modify global scope to declare class globals()[name] = clazz - return clazz() + return clazz(config) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index f73617f46..c67870aeb 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -2,11 +2,30 @@ IStrategy interface This module defines the interface to apply for strategies """ +import logging from abc import ABC, abstractmethod -from typing import Dict +from datetime import datetime +from enum import Enum +from typing import Dict, List, Tuple +import arrow from pandas import DataFrame +from freqtrade import constants +from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange import Exchange +from freqtrade.persistence import Trade + +logger = logging.getLogger(__name__) + + +class SignalType(Enum): + """ + Enum to distinguish between buy and sell signals + """ + BUY = "buy" + SELL = "sell" + class IStrategy(ABC): """ @@ -23,6 +42,9 @@ class IStrategy(ABC): stoploss: float ticker_interval: str + def __init__(self, config: dict): + self.config = config + @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ @@ -46,3 +68,169 @@ class IStrategy(ABC): :param dataframe: DataFrame :return: DataFrame with sell column """ + + def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: + """ + Parses the given ticker history and returns a populated DataFrame + add several TA indicators and buy signal to it + :return DataFrame with ticker data and indicator data + """ + dataframe = parse_ticker_dataframe(ticker_history) + dataframe = self.populate_indicators(dataframe) + dataframe = self.populate_buy_trend(dataframe) + dataframe = self.populate_sell_trend(dataframe) + return dataframe + + def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: + """ + Calculates current signal based several technical analysis indicators + :param pair: pair in format ANT/BTC + :param interval: Interval to use (in min) + :return: (Buy, Sell) A bool-tuple indicating buy/sell signal + """ + ticker_hist = exchange.get_ticker_history(pair, interval) + if not ticker_hist: + logger.warning('Empty ticker history for pair %s', pair) + return False, False + + try: + dataframe = self.analyze_ticker(ticker_hist) + except ValueError as error: + logger.warning( + 'Unable to analyze ticker for pair %s: %s', + pair, + str(error) + ) + return False, False + except Exception as error: + logger.exception( + 'Unexpected error when analyzing ticker for pair %s: %s', + pair, + str(error) + ) + return False, False + + if dataframe.empty: + logger.warning('Empty dataframe for pair %s', pair) + return False, False + + latest = dataframe.iloc[-1] + + # Check if dataframe is out of date + signal_date = arrow.get(latest['date']) + interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] + if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): + logger.warning( + 'Outdated history for pair %s. Last tick is %s minutes old', + pair, + (arrow.utcnow() - signal_date).seconds // 60 + ) + return False, False + + (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 + logger.debug( + 'trigger: %s (pair=%s) buy=%s sell=%s', + latest['date'], + pair, + str(buy), + str(sell) + ) + return buy, sell + + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: + """ + This function evaluate if on the condition required to trigger a sell has been reached + if the threshold is reached and updates the trade record. + :return: True if trade should be sold, False otherwise + """ + current_profit = trade.calc_profit_percent(rate) + if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, + current_profit=current_profit): + return True + + experimental = self.config.get('experimental', {}) + + if buy and experimental.get('ignore_roi_if_buy_signal', False): + logger.debug('Buy signal still active - not selling.') + return False + + # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) + if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): + logger.debug('Required profit reached. Selling..') + return True + + if experimental.get('sell_profit_only', False): + logger.debug('Checking if trade is profitable..') + if trade.calc_profit(rate=rate) <= 0: + return False + if sell and not buy and experimental.get('use_sell_signal', False): + logger.debug('Sell signal received. Selling..') + return True + + return False + + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, + current_profit: float) -> bool: + """ + Based on current profit of the trade and configured (trailing) stoploss, + decides to sell or not + """ + + trailing_stop = self.config.get('trailing_stop', False) + + trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) + + # evaluate if the stoploss was hit + if self.stoploss is not None and trade.stop_loss >= current_rate: + + if trailing_stop: + logger.debug( + f"HIT STOP: current price at {current_rate:.6f}, " + f"stop loss is {trade.stop_loss:.6f}, " + f"initial stop loss was at {trade.initial_stop_loss:.6f}, " + f"trade opened at {trade.open_rate:.6f}") + logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") + + logger.debug('Stop loss hit.') + return True + + # update the stop loss afterwards, after all by definition it's supposed to be hanging + if trailing_stop: + + # check if we have a special stop loss for positive condition + # and if profit is positive + stop_loss_value = self.stoploss + if 'trailing_stop_positive' in self.config and current_profit > 0: + + # Ignore mypy error check in configuration that this is a float + stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore + logger.debug(f"using positive stop loss mode: {stop_loss_value} " + f"since we have profit {current_profit}") + + trade.adjust_stop_loss(current_rate, stop_loss_value) + + return False + + def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: + """ + Based an earlier trade and current price and ROI configuration, decides whether bot should + sell + :return True if bot should sell at current rate + """ + + # Check if time matches and current rate is above threshold + time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 + for duration, threshold in self.minimal_roi.items(): + if time_diff <= duration: + return False + if current_profit > threshold: + return True + + return False + + def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: + """ + Creates a dataframe and populates indicators for given ticker data + """ + return {pair: self.populate_indicators(parse_ticker_dataframe(pair_data)) + for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 10cedb073..4df713b4d 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -34,6 +34,7 @@ class StrategyResolver(object): # Verify the strategy is in the configuration, otherwise fallback to the default strategy strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY self.strategy: IStrategy = self._load_strategy(strategy_name, + config=config, extra_dir=config.get('strategy_path')) # Set attributes @@ -62,10 +63,11 @@ class StrategyResolver(object): self.strategy.stoploss = float(self.strategy.stoploss) def _load_strategy( - self, strategy_name: str, extra_dir: Optional[str] = None) -> IStrategy: + self, strategy_name: str, config: dict, extra_dir: Optional[str] = None) -> IStrategy: """ Search and loads the specified strategy. :param strategy_name: name of the module to import + :param config: configuration for the strategy :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ @@ -81,10 +83,10 @@ class StrategyResolver(object): for path in abs_paths: try: - strategy = self._search_strategy(path, strategy_name) + strategy = self._search_strategy(path, strategy_name=strategy_name, config=config) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return import_strategy(strategy) + return import_strategy(strategy, config=config) except FileNotFoundError: logger.warning('Path "%s" does not exist', path) @@ -114,7 +116,7 @@ class StrategyResolver(object): return next(valid_strategies_gen, None) @staticmethod - def _search_strategy(directory: str, strategy_name: str) -> Optional[IStrategy]: + def _search_strategy(directory: str, strategy_name: str, config: dict) -> Optional[IStrategy]: """ Search for the strategy_name in the given directory :param directory: relative or absolute directory path @@ -130,5 +132,5 @@ class StrategyResolver(object): os.path.abspath(os.path.join(directory, entry)), strategy_name ) if strategy: - return strategy() + return strategy(config) return None diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 078dba447..788ab5afb 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -51,13 +51,13 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ # mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0}) patch_coinmarketcap(mocker, {'price_usd': 12345.0}) - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) +# mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) patch_exchange(mocker, None) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) - mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) +# mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) return FreqtradeBot(config) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 89f5a0bb7..6e4f91891 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -13,7 +13,6 @@ import pytest from arrow import Arrow from freqtrade import DependencyException, constants, optimize -from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) @@ -326,7 +325,6 @@ def test_backtesting_init(mocker, default_conf) -> None: get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf - assert isinstance(backtesting.analyze, Analyze) assert backtesting.ticker_interval == '5m' assert callable(backtesting.tickerdata_to_dataframe) assert callable(backtesting.populate_buy_trend) @@ -348,9 +346,9 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: data = backtesting.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 - # Load Analyze to compare the result between Backtesting function and Analyze are the same - analyze = Analyze(default_conf, DefaultStrategy()) - data2 = analyze.tickerdata_to_dataframe(tickerlist) + # Load strategy to compare the result between Backtesting function and strategy are the same + strategy = DefaultStrategy(default_conf) + data2 = strategy.tickerdata_to_dataframe(tickerlist) assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) @@ -413,7 +411,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') patch_exchange(mocker) @@ -454,7 +451,6 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') patch_exchange(mocker) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 6e59b4116..e6cfceae7 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -30,7 +30,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: """ Test rpc_trade_status() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -42,6 +41,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -74,7 +74,6 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: """ Test rpc_status_table() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -86,6 +85,7 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -108,7 +108,6 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, """ Test rpc_daily_profit() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -120,6 +119,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -160,7 +160,6 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, """ Test rpc_trade_statistics() method """ - patch_get_signal(mocker, (True, False)) mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -176,6 +175,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -237,7 +237,6 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, """ Test rpc_trade_statistics() method """ - patch_get_signal(mocker, (True, False)) mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -253,6 +252,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -309,7 +309,6 @@ def test_rpc_balance_handle(default_conf, mocker): } } - patch_get_signal(mocker, (True, False)) mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -323,6 +322,7 @@ def test_rpc_balance_handle(default_conf, mocker): ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) result = rpc._rpc_balance(default_conf['fiat_display_currency']) @@ -342,7 +342,6 @@ def test_rpc_start(mocker, default_conf) -> None: """ Test rpc_start() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -352,6 +351,7 @@ def test_rpc_start(mocker, default_conf) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -368,7 +368,6 @@ def test_rpc_stop(mocker, default_conf) -> None: """ Test rpc_stop() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -378,6 +377,7 @@ def test_rpc_stop(mocker, default_conf) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -395,7 +395,6 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: """ Test rpc_forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) @@ -417,6 +416,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -499,7 +499,6 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, """ Test rpc_performance() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -512,6 +511,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) # Create some test data @@ -538,7 +538,6 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: """ Test rpc_count() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -551,6 +550,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) trades = rpc._rpc_count() diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 01f248327..3336810bd 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -102,7 +102,6 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are authorized """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) patch_exchange(mocker, None) @@ -112,7 +111,9 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf)) + bot = FreqtradeBot(conf) + patch_get_signal(bot, (True, False)) + dummy = DummyCls(bot) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is True assert log_has( @@ -133,7 +134,6 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are unauthorized """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) patch_exchange(mocker, None) chat = Chat(0xdeadbeef, 0) @@ -142,7 +142,9 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf)) + bot = FreqtradeBot(conf) + patch_get_signal(bot, (True, False)) + dummy = DummyCls(bot) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -163,7 +165,6 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when an exception is thrown """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) patch_exchange(mocker) @@ -172,7 +173,11 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf)) + + bot = FreqtradeBot(conf) + patch_get_signal(bot, (True, False)) + dummy = DummyCls(bot) + dummy.dummy_exception(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -198,7 +203,6 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: conf['telegram']['enabled'] = False conf['telegram']['chat_id'] = 123 - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -233,6 +237,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -252,7 +257,6 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No """ Test _status() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -272,6 +276,8 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -299,7 +305,6 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) """ Test _status_table() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -320,6 +325,8 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) conf = deepcopy(default_conf) conf['stake_amount'] = 15.0 freqtradebot = FreqtradeBot(conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -353,7 +360,6 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, """ Test _daily() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch( 'freqtrade.fiat_convert.CryptoToFiatConverter._find_price', @@ -375,6 +381,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -427,7 +434,6 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: """ Test _daily() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -443,6 +449,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Try invalid data @@ -466,7 +473,6 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, """ Test _profit() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( @@ -485,6 +491,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) telegram._profit(bot=MagicMock(), update=update) @@ -568,7 +575,6 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: 'last': 0.1, } - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=mock_balance) mocker.patch('freqtrade.exchange.Exchange.get_ticker', side_effect=mock_ticker) @@ -581,6 +587,8 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -598,7 +606,6 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: """ Test _balance() method when the Exchange platform returns nothing """ - patch_get_signal(mocker, (True, False)) mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) msg_mock = MagicMock() @@ -609,6 +616,8 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -732,7 +741,6 @@ def test_forcesell_handle(default_conf, update, ticker, fee, """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -746,6 +754,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -785,7 +794,6 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -799,6 +807,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -842,7 +851,6 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -857,6 +865,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -891,7 +900,6 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) msg_mock = MagicMock() @@ -903,6 +911,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Trader is not running @@ -934,7 +943,6 @@ def test_performance_handle(default_conf, update, ticker, fee, """ Test _performance() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -951,6 +959,7 @@ def test_performance_handle(default_conf, update, ticker, fee, ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -976,7 +985,6 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: """ Test _performance() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -986,6 +994,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Trader is not running @@ -999,7 +1008,6 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non """ Test _count() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -1016,6 +1024,7 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non ) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 2175dc9b3..d965c3f20 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -23,7 +23,7 @@ def test_default_strategy_structure(): def test_default_strategy(result): - strategy = DefaultStrategy() + strategy = DefaultStrategy({}) assert type(strategy.minimal_roi) is dict assert type(strategy.stoploss) is float diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 1e082c380..0f879a67b 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -12,14 +12,15 @@ from freqtrade.strategy.resolver import StrategyResolver def test_import_strategy(caplog): caplog.set_level(logging.DEBUG) + default_config = {} - strategy = DefaultStrategy() + strategy = DefaultStrategy(default_config) strategy.some_method = lambda *args, **kwargs: 42 assert strategy.__module__ == 'freqtrade.strategy.default_strategy' assert strategy.some_method() == 42 - imported_strategy = import_strategy(strategy) + imported_strategy = import_strategy(strategy, default_config) assert dir(strategy) == dir(imported_strategy) @@ -35,13 +36,23 @@ def test_import_strategy(caplog): def test_search_strategy(): + default_config = {} default_location = os.path.join(os.path.dirname( os.path.realpath(__file__)), '..', '..', 'strategy' ) assert isinstance( - StrategyResolver._search_strategy(default_location, 'DefaultStrategy'), IStrategy + StrategyResolver._search_strategy( + default_location, + config=default_config, + strategy_name='DefaultStrategy' + ), + IStrategy ) - assert StrategyResolver._search_strategy(default_location, 'NotFoundStrategy') is None + assert StrategyResolver._search_strategy( + default_location, + config=default_config, + strategy_name='NotFoundStrategy' + ) is None def test_load_strategy(result): @@ -70,7 +81,7 @@ def test_load_not_found_strategy(): with pytest.raises(ImportError, match=r'Impossible to load Strategy \'NotFoundStrategy\'.' r' This class does not exist or contains Python code errors'): - strategy._load_strategy('NotFoundStrategy') + strategy._load_strategy(strategy_name='NotFoundStrategy', config={}) def test_strategy(result): diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 042422c5a..577fc3553 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,14 +10,14 @@ from unittest.mock import MagicMock import arrow from pandas import DataFrame -from freqtrade.analyze import Analyze, parse_ticker_dataframe +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has from freqtrade.strategy.default_strategy import DefaultStrategy # Avoid to reinit the same object again and again -_ANALYZE = Analyze({}, DefaultStrategy()) +#_ANALYZE = Analyze({}, DefaultStrategy()) def test_dataframe_correct_length(result): @@ -30,133 +30,133 @@ def test_dataframe_correct_columns(result): ['date', 'open', 'high', 'low', 'close', 'volume'] -def test_returns_latest_buy_signal(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) - ) - ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) +# def test_returns_latest_buy_signal(mocker, default_conf): +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) +# ) +# ) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) - ) - ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) +# ) +# ) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) -def test_returns_latest_sell_signal(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) - ) - ) +# def test_returns_latest_sell_signal(mocker, default_conf): +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) +# ) +# ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) - ) - ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) +# ) +# ) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) -def test_get_signal_empty(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) - exchange = get_patched_exchange(mocker, default_conf) - assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) - assert log_has('Empty ticker history for pair foo', caplog.record_tuples) +# def test_get_signal_empty(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) +# exchange = get_patched_exchange(mocker, default_conf) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) +# assert log_has('Empty ticker history for pair foo', caplog.record_tuples) -def test_get_signal_exception_valueerror(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - side_effect=ValueError('xyz') - ) - ) - assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) - assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) +# def test_get_signal_exception_valueerror(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# side_effect=ValueError('xyz') +# ) +# ) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) +# assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) -def test_get_signal_empty_dataframe(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([]) - ) - ) - assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) - assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) +# def test_get_signal_empty_dataframe(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([]) +# ) +# ) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) +# assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) -def test_get_signal_old_dataframe(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) - # default_conf defines a 5m interval. we check interval * 2 + 5m - # this is necessary as the last candle is removed (partial candles) by default - oldtime = arrow.utcnow().shift(minutes=-16) - ticks = DataFrame([{'buy': 1, 'date': oldtime}]) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame(ticks) - ) - ) - assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) - assert log_has( - 'Outdated history for pair xyz. Last tick is 16 minutes old', - caplog.record_tuples - ) +# def test_get_signal_old_dataframe(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) +# exchange = get_patched_exchange(mocker, default_conf) +# # default_conf defines a 5m interval. we check interval * 2 + 5m +# # this is necessary as the last candle is removed (partial candles) by default +# oldtime = arrow.utcnow().shift(minutes=-16) +# ticks = DataFrame([{'buy': 1, 'date': oldtime}]) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame(ticks) +# ) +# ) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) +# assert log_has( +# 'Outdated history for pair xyz. Last tick is 16 minutes old', +# caplog.record_tuples +# ) -def test_get_signal_handles_exceptions(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - side_effect=Exception('invalid ticker history ') - ) - ) +# def test_get_signal_handles_exceptions(mocker, default_conf): +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# side_effect=Exception('invalid ticker history ') +# ) +# ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) -def test_parse_ticker_dataframe(ticker_history): - columns = ['date', 'open', 'high', 'low', 'close', 'volume'] +# def test_parse_ticker_dataframe(ticker_history): +# columns = ['date', 'open', 'high', 'low', 'close', 'volume'] - # Test file with BV data - dataframe = parse_ticker_dataframe(ticker_history) - assert dataframe.columns.tolist() == columns +# # Test file with BV data +# dataframe = parse_ticker_dataframe(ticker_history) +# assert dataframe.columns.tolist() == columns -def test_tickerdata_to_dataframe(default_conf) -> None: - """ - Test Analyze.tickerdata_to_dataframe() method - """ - analyze = Analyze(default_conf, DefaultStrategy()) +# def test_tickerdata_to_dataframe(default_conf) -> None: +# """ +# Test Analyze.tickerdata_to_dataframe() method +# """ +# analyze = Analyze(default_conf, DefaultStrategy()) - timerange = TimeRange(None, 'line', 0, -100) - tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': tick} - data = analyze.tickerdata_to_dataframe(tickerlist) - assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed +# timerange = TimeRange(None, 'line', 0, -100) +# tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) +# tickerlist = {'UNITTEST/BTC': tick} +# data = analyze.tickerdata_to_dataframe(tickerlist) +# assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index f4b26eb1d..019587af1 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -2,7 +2,6 @@ import pandas -from freqtrade.analyze import Analyze from freqtrade.optimize import load_data from freqtrade.strategy.resolver import StrategyResolver @@ -15,8 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - analyze = Analyze({}, strategy) - dataframe = analyze.analyze_ticker(dataframe) + dataframe = strategy.analyze_ticker(dataframe) return dataframe diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c628a9da3..b7ae96048 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -31,7 +31,6 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: :param config: Config to pass to the bot :return: None """ - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) patch_exchange(mocker) @@ -40,17 +39,13 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: return FreqtradeBot(config) -def patch_get_signal(mocker, value=(True, False)) -> None: +def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: """ - - :param mocker: mocker to patch Analyze class - :param value: which value Analyze.get_signal() must return + :param mocker: mocker to patch IStrategy class + :param value: which value IStrategy.get_signal() must return :return: None """ - mocker.patch( - 'freqtrade.freqtradebot.Analyze.get_signal', - side_effect=lambda e, s, t: value - ) + freqtrade.strategy.get_signal = lambda e, s, t: value def patch_RPCManager(mocker) -> MagicMock: @@ -267,7 +262,6 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, """ Test get_trade_stake_amount() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -285,6 +279,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, conf['max_open_trades'] = 2 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) # no open trades, order amount should be 'balance / max_open_trades' result = freqtrade._get_trade_stake_amount() @@ -452,7 +447,6 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -467,6 +461,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke # Save state of current whitelist whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) freqtrade.create_trade() trade = Trade.query.first() @@ -490,7 +485,6 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -503,6 +497,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade.create_trade() @@ -513,7 +508,6 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -529,6 +523,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, conf = deepcopy(default_conf) conf['stake_amount'] = 0.0005 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) freqtrade.create_trade() rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] @@ -540,7 +535,6 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -556,6 +550,7 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord conf = deepcopy(default_conf) conf['stake_amount'] = 0.000000005 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) result = freqtrade.create_trade() assert result is False @@ -566,7 +561,6 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -583,6 +577,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) assert freqtrade.create_trade() is False assert freqtrade._get_trade_stake_amount() is None @@ -592,7 +587,6 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -608,6 +602,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) freqtrade.create_trade() @@ -620,7 +615,6 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -636,6 +630,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) freqtrade.create_trade() @@ -650,7 +645,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: conf = deepcopy(default_conf) conf['dry_run'] = True - patch_get_signal(mocker, value=(False, False)) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -664,6 +658,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: conf = deepcopy(default_conf) conf['stake_amount'] = 10 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade, value=(False, False)) Trade.query = MagicMock() Trade.query.filter = MagicMock() @@ -675,7 +670,6 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, """ Test the trade creation in _process() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -688,6 +682,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -716,7 +711,6 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non """ Test _process() method when a RequestException happens """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -729,6 +723,8 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + result = freqtrade._process() assert result is False assert sleep_mock.has_calls() @@ -738,7 +734,6 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> """ Test _process() method when an OperationalException happens """ - patch_get_signal(mocker) msg_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -749,6 +744,8 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> buy=MagicMock(side_effect=OperationalException) ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + assert freqtrade.state == State.RUNNING result = freqtrade._process() @@ -762,7 +759,6 @@ def test_process_trade_handling( """ Test _process() """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -775,6 +771,7 @@ def test_process_trade_handling( get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -913,7 +910,6 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, """ Test check_handle() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -931,6 +927,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) freqtrade.create_trade() @@ -941,7 +938,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, trade.update(limit_buy_order) assert trade.is_open is True - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.open_order_id == limit_sell_order['id'] @@ -962,10 +959,8 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) - patch_get_signal(mocker, value=(True, True)) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -976,6 +971,8 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, ) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade, value=(True, True)) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -985,7 +982,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert nb_trades == 0 # Buy is triggering, so buying ... - patch_get_signal(mocker, value=(True, False)) + patch_get_signal(freqtrade, value=(True, False)) freqtrade.create_trade() trades = Trade.query.all() nb_trades = len(trades) @@ -993,7 +990,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert trades[0].is_open is True # Buy and Sell are not triggering, so doing nothing ... - patch_get_signal(mocker, value=(False, False)) + patch_get_signal(freqtrade, value=(False, False)) assert freqtrade.handle_trade(trades[0]) is False trades = Trade.query.all() nb_trades = len(trades) @@ -1001,7 +998,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert trades[0].is_open is True # Buy and Sell are triggering, so doing nothing ... - patch_get_signal(mocker, value=(True, True)) + patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trades[0]) is False trades = Trade.query.all() nb_trades = len(trades) @@ -1009,7 +1006,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert trades[0].is_open is True # Sell is triggering, guess what : we are Selling! - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) trades = Trade.query.all() assert freqtrade.handle_trade(trades[0]) is True @@ -1023,7 +1020,6 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) - patch_get_signal(mocker, value=(True, False)) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1035,8 +1031,10 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, get_markets=markets ) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade, value=(True, False)) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.create_trade() trade = Trade.query.first() @@ -1047,7 +1045,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, # we might just want to check if we are in a sell condition without # executing # if ROI is reached we must sell - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) assert log_has('Required profit reached. Selling..', caplog.record_tuples) @@ -1061,7 +1059,6 @@ def test_handle_trade_experimental( conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1072,18 +1069,19 @@ def test_handle_trade_experimental( get_fee=fee, get_markets=markets ) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() trade = Trade.query.first() trade.is_open = True - patch_get_signal(mocker, value=(False, False)) + patch_get_signal(freqtrade, value=(False, False)) assert not freqtrade.handle_trade(trade) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) assert log_has('Sell signal received. Selling..', caplog.record_tuples) @@ -1093,7 +1091,6 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, """ Test check_handle() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1105,6 +1102,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create trade and sell it freqtrade.create_trade() @@ -1345,7 +1343,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc """ Test execute_sell() method with a ticker going UP """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1357,6 +1354,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1397,7 +1395,6 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, """ Test execute_sell() method with a ticker going DOWN """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) @@ -1409,6 +1406,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1450,7 +1448,6 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, """ Test execute_sell() method with a ticker going DOWN and with a bot config empty """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -1461,6 +1458,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1500,7 +1498,6 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, """ Test execute_sell() method with a ticker going DOWN and with a bot config empty """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -1511,6 +1508,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1550,10 +1548,8 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, """ Test sell_profit_only feature when enabled """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1572,11 +1568,14 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, 'sell_profit_only': True, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1585,10 +1584,8 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, """ Test sell_profit_only feature when disabled """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1607,11 +1604,13 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, 'sell_profit_only': False, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1619,10 +1618,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.stop_loss_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1641,11 +1638,14 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market 'sell_profit_only': True, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.stop_loss_reached = \ + lambda current_rate, trade, current_time, current_profit: False freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is False @@ -1653,10 +1653,8 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1677,11 +1675,14 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1689,10 +1690,8 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1712,15 +1711,18 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(True, True)) + patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trade) is False # Test if buy-signal is absent (should sell due to roi = true) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1728,10 +1730,8 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1748,6 +1748,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) conf['trailing_stop'] = True print(limit_buy_order) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() trade = Trade.query.first() @@ -1765,10 +1768,8 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, Test sell_profit_only feature when enabled and we have a loss """ buy_price = limit_buy_order['price'] - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1785,6 +1786,8 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, conf['trailing_stop'] = True conf['trailing_stop_positive'] = 0.01 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() trade = Trade.query.first() @@ -1826,10 +1829,8 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1849,16 +1850,19 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) # Sell due to min_roi_reached - patch_get_signal(mocker, value=(True, True)) + patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trade) is True # Test if buy-signal is absent - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1869,7 +1873,6 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1882,6 +1885,8 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1896,7 +1901,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1909,6 +1913,8 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1922,7 +1928,6 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo """ trades_for_order[0]['fee']['currency'] = 'ETH' - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1936,6 +1941,8 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1948,7 +1955,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock trades_for_order[0]['fee']['currency'] = 'BNB' trades_for_order[0]['fee']['cost'] = 0.00094518 - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1962,6 +1968,8 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1971,7 +1979,6 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c Test get_real_amount with split trades (multiple trades for this order) """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1985,6 +1992,8 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1999,7 +2008,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2014,6 +2022,8 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -2028,7 +2038,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2042,6 +2051,8 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -2053,7 +2064,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, # Remove "Currency" from fee dict trades_for_order[0]['fee'] = {'cost': 0.008} - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2067,6 +2077,7 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -2075,7 +2086,6 @@ def test_get_real_amount_open_trade(default_conf, mocker): """ Test get_real_amount condition trade.fee_open == 0 or order['status'] == 'open' """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2093,4 +2103,5 @@ def test_get_real_amount_open_trade(default_conf, mocker): 'status': 'open', } freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) assert freqtrade.get_real_amount(trade, order) == amount diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 15ed1550b..1e9b5c181 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -7,7 +7,7 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock -from freqtrade.analyze import Analyze, parse_ticker_dataframe +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file @@ -48,10 +48,10 @@ def test_common_datearray(default_conf) -> None: Test common_datearray() :return: None """ - analyze = Analyze(default_conf, DefaultStrategy()) + strategy = DefaultStrategy(default_conf) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = analyze.tickerdata_to_dataframe(tickerlist) + dataframes = strategy.tickerdata_to_dataframe(tickerlist) dates = common_datearray(dataframes) From 5c87c420c7d4f0f1f60367a2c85b13b8ceb62957 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 16 Jul 2018 08:59:14 +0300 Subject: [PATCH 19/47] restore one analyze test --- freqtrade/tests/test_analyze.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 577fc3553..8411fa250 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -141,12 +141,12 @@ def test_dataframe_correct_columns(result): # assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) -# def test_parse_ticker_dataframe(ticker_history): -# columns = ['date', 'open', 'high', 'low', 'close', 'volume'] +def test_parse_ticker_dataframe(ticker_history): + columns = ['date', 'open', 'high', 'low', 'close', 'volume'] -# # Test file with BV data -# dataframe = parse_ticker_dataframe(ticker_history) -# assert dataframe.columns.tolist() == columns + # Test file with BV data + dataframe = parse_ticker_dataframe(ticker_history) + assert dataframe.columns.tolist() == columns # def test_tickerdata_to_dataframe(default_conf) -> None: From 78af4bc78506e00a2b1d563db8ca8e0b44e114bb Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 10:21:53 +0300 Subject: [PATCH 20/47] move and fix tests from Analyze to interface of strategy --- freqtrade/tests/strategy/test_interface.py | 126 +++++++++++++++++++ freqtrade/tests/test_analyze.py | 137 --------------------- 2 files changed, 126 insertions(+), 137 deletions(-) create mode 100644 freqtrade/tests/strategy/test_interface.py diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py new file mode 100644 index 000000000..a016b7f96 --- /dev/null +++ b/freqtrade/tests/strategy/test_interface.py @@ -0,0 +1,126 @@ +# pragma pylint: disable=missing-docstring, C0103 + +""" +Unit test file for analyse.py +""" + +import logging +from unittest.mock import MagicMock + +import arrow +from pandas import DataFrame + +from freqtrade.arguments import TimeRange +from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.strategy.default_strategy import DefaultStrategy + +# Avoid to reinit the same object again and again +_STRATEGY = DefaultStrategy(config={}) + + +def test_returns_latest_buy_signal(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) + + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) + + +def test_returns_latest_sell_signal(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) + ) + + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) + + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) + + +def test_get_signal_empty(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) + exchange = get_patched_exchange(mocker, default_conf) + assert (False, False) == _STRATEGY.get_signal(exchange, 'foo', default_conf['ticker_interval']) + assert log_has('Empty ticker history for pair foo', caplog.record_tuples) + + +def test_get_signal_exception_valueerror(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + side_effect=ValueError('xyz') + ) + assert (False, False) == _STRATEGY.get_signal(exchange, 'foo', default_conf['ticker_interval']) + assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) + + +def test_get_signal_empty_dataframe(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([]) + ) + assert (False, False) == _STRATEGY.get_signal(exchange, 'xyz', default_conf['ticker_interval']) + assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) + + +def test_get_signal_old_dataframe(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) + # default_conf defines a 5m interval. we check interval * 2 + 5m + # this is necessary as the last candle is removed (partial candles) by default + oldtime = arrow.utcnow().shift(minutes=-16) + ticks = DataFrame([{'buy': 1, 'date': oldtime}]) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame(ticks) + ) + assert (False, False) == _STRATEGY.get_signal(exchange, 'xyz', default_conf['ticker_interval']) + assert log_has( + 'Outdated history for pair xyz. Last tick is 16 minutes old', + caplog.record_tuples + ) + + +def test_get_signal_handles_exceptions(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + side_effect=Exception('invalid ticker history ') + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) + + +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Analyze.tickerdata_to_dataframe() method + """ + strategy = DefaultStrategy(default_conf) + + timerange = TimeRange(None, 'line', 0, -100) + tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) + tickerlist = {'UNITTEST/BTC': tick} + data = strategy.tickerdata_to_dataframe(tickerlist) + assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 8411fa250..fdd83809c 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -4,20 +4,7 @@ Unit test file for analyse.py """ -import logging -from unittest.mock import MagicMock - -import arrow -from pandas import DataFrame - from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.arguments import TimeRange -from freqtrade.optimize.__init__ import load_tickerdata_file -from freqtrade.tests.conftest import get_patched_exchange, log_has -from freqtrade.strategy.default_strategy import DefaultStrategy - -# Avoid to reinit the same object again and again -#_ANALYZE = Analyze({}, DefaultStrategy()) def test_dataframe_correct_length(result): @@ -30,133 +17,9 @@ def test_dataframe_correct_columns(result): ['date', 'open', 'high', 'low', 'close', 'volume'] -# def test_returns_latest_buy_signal(mocker, default_conf): -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) -# ) -# ) -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) - -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) -# ) -# ) -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) - - -# def test_returns_latest_sell_signal(mocker, default_conf): -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) -# ) -# ) - -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) - -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) -# ) -# ) -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) - - -# def test_get_signal_empty(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) -# exchange = get_patched_exchange(mocker, default_conf) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) -# assert log_has('Empty ticker history for pair foo', caplog.record_tuples) - - -# def test_get_signal_exception_valueerror(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# side_effect=ValueError('xyz') -# ) -# ) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) -# assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) - - -# def test_get_signal_empty_dataframe(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([]) -# ) -# ) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) -# assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) - - -# def test_get_signal_old_dataframe(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) -# exchange = get_patched_exchange(mocker, default_conf) -# # default_conf defines a 5m interval. we check interval * 2 + 5m -# # this is necessary as the last candle is removed (partial candles) by default -# oldtime = arrow.utcnow().shift(minutes=-16) -# ticks = DataFrame([{'buy': 1, 'date': oldtime}]) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame(ticks) -# ) -# ) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) -# assert log_has( -# 'Outdated history for pair xyz. Last tick is 16 minutes old', -# caplog.record_tuples -# ) - - -# def test_get_signal_handles_exceptions(mocker, default_conf): -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# side_effect=Exception('invalid ticker history ') -# ) -# ) - -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) - - def test_parse_ticker_dataframe(ticker_history): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] # Test file with BV data dataframe = parse_ticker_dataframe(ticker_history) assert dataframe.columns.tolist() == columns - - -# def test_tickerdata_to_dataframe(default_conf) -> None: -# """ -# Test Analyze.tickerdata_to_dataframe() method -# """ -# analyze = Analyze(default_conf, DefaultStrategy()) - -# timerange = TimeRange(None, 'line', 0, -100) -# tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) -# tickerlist = {'UNITTEST/BTC': tick} -# data = analyze.tickerdata_to_dataframe(tickerlist) -# assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed From dbc3874b4f41ff7d4b99cf257dc2d706f4679316 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 10:47:15 +0300 Subject: [PATCH 21/47] __init__ must return None to please mypy --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c67870aeb..e9d5b904d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -42,7 +42,7 @@ class IStrategy(ABC): stoploss: float ticker_interval: str - def __init__(self, config: dict): + def __init__(self, config: dict) -> None: self.config = config @abstractmethod From 084264669fc936ec6ba22a28d4cb5fd255fd8159 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:02:07 +0300 Subject: [PATCH 22/47] fix the last failing unit test --- freqtrade/tests/strategy/test_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 0f879a67b..52021475a 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -64,7 +64,7 @@ def test_load_strategy(result): def test_load_strategy_invalid_directory(result, caplog): resolver = StrategyResolver() extra_dir = os.path.join('some', 'path') - resolver._load_strategy('TestStrategy', extra_dir) + resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir) assert ( 'freqtrade.strategy.resolver', From 06d024cc46f56a118a459537bc1e85c8adf3546c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:07:27 +0300 Subject: [PATCH 23/47] make pytest ignore this file --- user_data/strategies/test_strategy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 34f496e38..c04f4935f 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -12,6 +12,7 @@ import numpy # noqa # This class is a sample. Feel free to customize it. class TestStrategy(IStrategy): + __test__ = False # pytest expects to find tests here because of the name """ This is a test strategy to inspire you. More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md From e11ec2896255d9e0ac52548170a46a2316017d5a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:13:35 +0300 Subject: [PATCH 24/47] remove leftover commented-out code --- freqtrade/freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0d916f570..222e6ec96 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -50,7 +50,6 @@ class FreqtradeBot(object): # Init objects self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy -# self.analyze = Analyze(self.config, self.strategy) self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None From 50b15b8052b7c096e7623d681486d5c83ebf55e3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:41:21 +0300 Subject: [PATCH 25/47] fix plot_dataframe to use strategy instead of Analyze --- scripts/plot_dataframe.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 3b86afc9e..11f1f85d5 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -40,11 +40,11 @@ from plotly.offline import plot import freqtrade.optimize as optimize from freqtrade import persistence -from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade +from freqtrade.strategy.resolver import StrategyResolver logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} @@ -122,7 +122,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Load the strategy try: - analyze = Analyze(_CONF) + strategy = StrategyResolver(_CONF).strategy exchange = Exchange(_CONF) except AttributeError: logger.critical( @@ -132,7 +132,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exit() # Set the ticker to use - tick_interval = analyze.get_ticker_interval() + tick_interval = strategy.ticker_interval # Load pair tickers tickers = {} @@ -156,11 +156,11 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Get trades already made from the DB trades = load_trades(args, pair, timerange) - dataframes = analyze.tickerdata_to_dataframe(tickers) + dataframes = strategy.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] - dataframe = analyze.populate_buy_trend(dataframe) - dataframe = analyze.populate_sell_trend(dataframe) + dataframe = strategy.populate_buy_trend(dataframe) + dataframe = strategy.populate_sell_trend(dataframe) if len(dataframe.index) > args.plot_limit: logger.warning('Ticker contained more than %s candles as defined ' From 4a26eb34eab381c1a7b57cbd8148439509256c6c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:47:09 +0300 Subject: [PATCH 26/47] fix plot_profit to use strategy instead of Analyze --- scripts/plot_profit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 012446065..9c3468c74 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -26,9 +26,8 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.analyze import Analyze from freqtrade import constants - +from freqtrade.strategy.resolver import StrategyResolver import freqtrade.optimize as optimize import freqtrade.misc as misc @@ -87,7 +86,8 @@ def plot_profit(args: Namespace) -> None: # Init strategy try: - analyze = Analyze({'strategy': config.get('strategy')}) + strategy = StrategyResolver({'strategy': config.get('strategy')}).strategy + except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', @@ -113,7 +113,7 @@ def plot_profit(args: Namespace) -> None: else: filter_pairs = config['exchange']['pair_whitelist'] - tick_interval = analyze.strategy.ticker_interval + tick_interval = strategy.ticker_interval pairs = config['exchange']['pair_whitelist'] if filter_pairs: @@ -127,7 +127,7 @@ def plot_profit(args: Namespace) -> None: refresh_pairs=False, timerange=timerange ) - dataframes = analyze.tickerdata_to_dataframe(tickers) + dataframes = strategy.tickerdata_to_dataframe(tickers) # NOTE: the dataframes are of unequal length, # 'dates' is an merged date array of them all. From 85fd4dd3ff580e15e3b489753005000842dcc017 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 12:12:25 +0300 Subject: [PATCH 27/47] rename analyze.py to exchange_helpers.py --- freqtrade/{analyze.py => exchange/exchange_helpers.py} | 0 freqtrade/strategy/interface.py | 2 +- freqtrade/tests/conftest.py | 6 ++---- .../{test_analyze.py => exchange/test_exchange_helpers.py} | 4 ++-- freqtrade/tests/strategy/test_default_strategy.py | 2 +- freqtrade/tests/test_misc.py | 2 +- 6 files changed, 7 insertions(+), 9 deletions(-) rename freqtrade/{analyze.py => exchange/exchange_helpers.py} (100%) rename freqtrade/tests/{test_analyze.py => exchange/test_exchange_helpers.py} (85%) diff --git a/freqtrade/analyze.py b/freqtrade/exchange/exchange_helpers.py similarity index 100% rename from freqtrade/analyze.py rename to freqtrade/exchange/exchange_helpers.py diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e9d5b904d..fb8bcd31d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -12,7 +12,7 @@ import arrow from pandas import DataFrame from freqtrade import constants -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.persistence import Trade diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 788ab5afb..a9ed92765 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,7 +12,7 @@ from jsonschema import validate from telegram import Chat, Message, Update from freqtrade import constants -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot @@ -20,7 +20,7 @@ logging.getLogger('').setLevel(logging.INFO) def log_has(line, logs): - # caplog mocker returns log as a tuple: ('freqtrade.analyze', logging.WARNING, 'foobar') + # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar') # and we want to match line against foobar in the tuple return reduce(lambda a, b: a or b, filter(lambda x: x[2] == line, logs), @@ -51,13 +51,11 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ # mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0}) patch_coinmarketcap(mocker, {'price_usd': 12345.0}) -# mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) patch_exchange(mocker, None) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) -# mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) return FreqtradeBot(config) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/exchange/test_exchange_helpers.py similarity index 85% rename from freqtrade/tests/test_analyze.py rename to freqtrade/tests/exchange/test_exchange_helpers.py index fdd83809c..6a3bc9eb6 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/exchange/test_exchange_helpers.py @@ -1,10 +1,10 @@ # pragma pylint: disable=missing-docstring, C0103 """ -Unit test file for analyse.py +Unit test file for exchange_helpers.py """ -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe def test_dataframe_correct_length(result): diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index d965c3f20..37df1748f 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -3,7 +3,7 @@ import json import pytest from pandas import DataFrame -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.strategy.default_strategy import DefaultStrategy diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 1e9b5c181..76290c6ca 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -7,7 +7,7 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file From e17618407bc3abb75eaed957b2faf350e45088a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 20:26:59 +0200 Subject: [PATCH 28/47] Rename --realistic-simulation to --enable-position-stacking --- docs/bot-usage.md | 8 +++---- freqtrade/arguments.py | 7 ++++--- freqtrade/configuration.py | 11 +++++----- freqtrade/optimize/backtesting.py | 13 ++++++------ freqtrade/optimize/hyperopt.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 22 ++++++++++---------- freqtrade/tests/test_configuration.py | 10 ++++----- 7 files changed, 38 insertions(+), 35 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 25fc78f0a..a02403160 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -127,8 +127,8 @@ optional arguments: -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) --realistic-simulation - uses max_open_trades from config to simulate real - world limitations + Disables buying the same pair multiple times to + simulate real world limitations --timerange TIMERANGE specify what timerange of data to use. -l, --live using live data @@ -173,8 +173,8 @@ optional arguments: -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) --realistic-simulation - uses max_open_trades from config to simulate real - world limitations + Disables buying the same pair multiple times to + simulate real world limitations --timerange TIMERANGE specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 731c5d88c..2c1d05070 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -178,10 +178,11 @@ class Arguments(object): type=str, ) parser.add_argument( - '--realistic-simulation', - help='uses max_open_trades from config to simulate real world limitations', + '--enable-position-stacking', + help='Allow buying the same pair twice (position stacking)', action='store_true', - dest='realistic_simulation', + dest='position_stacking', + default=False ) parser.add_argument( '--timerange', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 582b2889c..276156b8d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -142,10 +142,11 @@ class Configuration(object): config.update({'live': True}) logger.info('Parameter -l/--live detected ...') - # If --realistic-simulation is used we add it to the configuration - if 'realistic_simulation' in self.args and self.args.realistic_simulation: - config.update({'realistic_simulation': True}) - logger.info('Parameter --realistic-simulation detected ...') + # If --enable-position-stacking is used we add it to the configuration + if 'position_stacking' in self.args and self.args.position_stacking: + config.update({'position_stacking': True}) + logger.info('Parameter --enable-position-stacking detected ...') + logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) # If --timerange is used we add it to the configuration @@ -182,7 +183,7 @@ class Configuration(object): Extract information for sys.argv and load Hyperopt configuration :return: configuration as dictionary """ - # If --realistic-simulation is used we add it to the configuration + # If --epochs is used we add it to the configuration if 'epochs' in self.args and self.args.epochs: config.update({'epochs': self.args.epochs}) logger.info('Parameter --epochs detected ...') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..9c336db6d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -198,13 +198,13 @@ class Backtesting(object): stake_amount: btc amount to use for each trade processed: a processed dictionary with format {pair, data} max_open_trades: maximum number of concurrent trades (default: 0, disabled) - realistic: do we try to simulate realistic trades? (default: True) + position_stacking: do we allow position stacking? (default: False) :return: DataFrame """ headers = ['date', 'buy', 'open', 'close', 'sell'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) + position_stacking = args.get('position_stacking', False) trades = [] trade_count_lock: Dict = {} for pair, pair_data in processed.items(): @@ -228,7 +228,7 @@ class Backtesting(object): if row.buy == 0 or row.sell == 1: continue # skip rows where no buy signal or that would immediately sell off - if realistic: + if not position_stacking: if lock_pair_until is not None and row.date <= lock_pair_until: continue if max_open_trades > 0: @@ -283,10 +283,11 @@ class Backtesting(object): logger.critical("No data found. Terminating.") return # Ignore max_open_trades in backtesting, except realistic flag was passed - if self.config.get('realistic_simulation', False): + # TODO: this is not position stacking!! + if self.config.get('position_stacking', False): max_open_trades = self.config['max_open_trades'] else: - logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') + logger.info('Ignoring max_open_trades (position_stacking not set) ...') max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) @@ -306,7 +307,7 @@ class Backtesting(object): 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, - 'realistic': self.config.get('realistic_simulation', False), + 'position_stacking': self.config.get('position_stacking', False), } ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 72bf34eb3..73b74e6ad 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -280,7 +280,7 @@ class Hyperopt(Backtesting): { 'stake_amount': self.config['stake_amount'], 'processed': processed, - 'realistic': self.config.get('realistic_simulation', False), + 'position_stacking': self.config.get('position_stacking', False), } ) result_explanation = self.format_results(results) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6fbf71e40..2c559d68e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -96,7 +96,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: 'stake_amount': config['stake_amount'], 'processed': processed, 'max_open_trades': 1, - 'realistic': True + 'position_stacking': False } ) # results :: @@ -127,7 +127,7 @@ def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): 'stake_amount': conf['stake_amount'], 'processed': backtesting.tickerdata_to_dataframe(data), 'max_open_trades': 10, - 'realistic': True, + 'position_stacking': False, 'record': record } @@ -193,8 +193,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'live' not in config assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation' not in config - assert not log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking' not in config + assert not log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert 'refresh_pairs' not in config assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) @@ -218,7 +218,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'backtesting', '--ticker-interval', '1m', '--live', - '--realistic-simulation', + '--enable-position-stacking', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo', @@ -246,8 +246,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'live' in config assert log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation' in config - assert log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking' in config + assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) assert 'refresh_pairs' in config @@ -495,7 +495,7 @@ def test_backtest(default_conf, fee, mocker) -> None: 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, 'max_open_trades': 10, - 'realistic': True + 'position_stacking': False } ) assert not results.empty @@ -543,7 +543,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: 'stake_amount': default_conf['stake_amount'], 'processed': backtesting.tickerdata_to_dataframe(data), 'max_open_trades': 1, - 'realistic': True + 'position_stacking': False } ) assert not results.empty @@ -718,7 +718,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): '--ticker-interval', '1m', '--live', '--timerange', '-100', - '--realistic-simulation' + '--enable-position-stacking' ] args = get_args(args) start(args) @@ -734,7 +734,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', - 'Parameter --realistic-simulation detected ...' + 'Parameter --enable-position-stacking detected ...' ] for line in exists: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e64e1b486..d29e4e9d9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -275,8 +275,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'live' not in config assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation' not in config - assert not log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking' not in config + assert not log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert 'refresh_pairs' not in config assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) @@ -300,7 +300,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'backtesting', '--ticker-interval', '1m', '--live', - '--realistic-simulation', + '--enable-position-stacking', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo' @@ -330,8 +330,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'live' in config assert log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation'in config - assert log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking'in config + assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) assert 'refresh_pairs'in config From b29eed32ca2c7ee2c1116c5d018b6879f9069fc5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 20:29:53 +0200 Subject: [PATCH 29/47] update documentation --- docs/backtesting.md | 8 ++++---- docs/bot-usage.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 172969ae2..5044c9243 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -29,25 +29,25 @@ The backtesting is very easy with freqtrade. #### With 5 min tickers (Per default) ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation +python3 ./freqtrade/main.py backtesting ``` #### With 1 min tickers ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation --ticker-interval 1m +python3 ./freqtrade/main.py backtesting --ticker-interval 1m ``` #### Update cached pairs with the latest data ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-cached +python3 ./freqtrade/main.py backtesting --refresh-pairs-cached ``` #### With live data (do not alter your testdata files) ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation --live +python3 ./freqtrade/main.py backtesting --live ``` #### Using a different on-disk ticker-data source diff --git a/docs/bot-usage.md b/docs/bot-usage.md index a02403160..1b83ff8e7 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -126,7 +126,7 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --realistic-simulation + --enable-position-tracking Disables buying the same pair multiple times to simulate real world limitations --timerange TIMERANGE @@ -164,7 +164,7 @@ To optimize your strategy, you can use hyperopt parameter hyperoptimization to find optimal parameter values for your stategy. ``` -usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] +usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--enable-position-tracking] [--timerange TIMERANGE] [-e INT] [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] @@ -172,7 +172,7 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --realistic-simulation + --enable-position-tracking Disables buying the same pair multiple times to simulate real world limitations --timerange TIMERANGE specify what timerange of data to use. From c82276ecbeaa056204ebbac40575f425dc98ee17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 21:05:03 +0200 Subject: [PATCH 30/47] add --disable-max-market-positions --- freqtrade/arguments.py | 10 ++++++++++ freqtrade/configuration.py | 8 +++++++- freqtrade/optimize/backtesting.py | 7 +++---- freqtrade/tests/optimize/test_backtesting.py | 11 ++++++++--- freqtrade/tests/test_configuration.py | 6 +++++- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 2c1d05070..3b10b90df 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -184,6 +184,16 @@ class Arguments(object): dest='position_stacking', default=False ) + + parser.add_argument( + '--disable-max-market-positions', + help='Disable applying `max_open_trades` during backtest ' + '(same as setting `max_open_trades` to a very high number)', + action='store_false', + dest='use_max_market_positions', + default=True + ) + parser.add_argument( '--timerange', help='specify what timerange of data to use.', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 276156b8d..f5c1c398d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -147,7 +147,13 @@ class Configuration(object): config.update({'position_stacking': True}) logger.info('Parameter --enable-position-stacking detected ...') - logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) + # If --disable-max-market-positions is used we add it to the configuration + if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions: + config.update({'use_max_market_positions': False}) + logger.info('Parameter --disable-max-market-positions detected ...') + logger.info('max_open_trades set to unlimited ...') + else: + logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9c336db6d..b050af8b5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -282,12 +282,11 @@ class Backtesting(object): if not data: logger.critical("No data found. Terminating.") return - # Ignore max_open_trades in backtesting, except realistic flag was passed - # TODO: this is not position stacking!! - if self.config.get('position_stacking', False): + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: - logger.info('Ignoring max_open_trades (position_stacking not set) ...') + logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 2c559d68e..dc4b667df 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -219,6 +219,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--ticker-interval', '1m', '--live', '--enable-position-stacking', + '--disable-max-market-positions', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo', @@ -248,7 +249,10 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'position_stacking' in config assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) - assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'use_max_market_positions' in config + assert log_has('Parameter --disable-max-market-positions detected ...', caplog.record_tuples) + assert log_has('max_open_trades set to unlimited ...', caplog.record_tuples) assert 'refresh_pairs' in config assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) @@ -718,7 +722,8 @@ def test_backtest_start_live(default_conf, mocker, caplog): '--ticker-interval', '1m', '--live', '--timerange', '-100', - '--enable-position-stacking' + '--enable-position-stacking', + '--disable-max-market-positions' ] args = get_args(args) start(args) @@ -727,7 +732,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Parameter -i/--ticker-interval detected ...', 'Using ticker_interval: 1m ...', 'Parameter -l/--live detected ...', - 'Using max_open_trades: 1 ...', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: -100 ...', 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d29e4e9d9..da8d3bebf 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -301,6 +301,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--ticker-interval', '1m', '--live', '--enable-position-stacking', + '--disable-max-market-positions', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo' @@ -332,7 +333,10 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'position_stacking'in config assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) - assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'use_max_market_positions' in config + assert log_has('Parameter --disable-max-market-positions detected ...', caplog.record_tuples) + assert log_has('max_open_trades set to unlimited ...', caplog.record_tuples) assert 'refresh_pairs'in config assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) From a290286fef14450c0c5f69084aad0e39242ab28f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 21:05:31 +0200 Subject: [PATCH 31/47] update documentation --- docs/bot-usage.md | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 1b83ff8e7..d5c76f4a4 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -117,18 +117,22 @@ python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.s Backtesting also uses the config specified via `-c/--config`. ``` -usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--realistic-simulation] - [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] - [--export-filename EXPORTFILENAME] - +usage: main.py backtesting [-h] [-i TICKER_INTERVAL] + [--enable-position-stacking] + [--disable-max-market-positions] + [--timerange TIMERANGE] [-l] [-r] + [--export EXPORT] [--export-filename PATH] optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-tracking - Disables buying the same pair multiple times to - simulate real world limitations + --enable-position-stacking + Allow buying the same pair twice (position stacking) + --disable-max-market-positions + Disable applying `max_open_trades` during backtest + (same as setting `max_open_trades` to a very high + number) --timerange TIMERANGE specify what timerange of data to use. -l, --live using live data @@ -138,11 +142,13 @@ optional arguments: run your backtesting with up-to-date data. --export EXPORT export backtest results, argument are: trades Example --export=trades - --export-filename EXPORTFILENAME + --export-filename PATH Save backtest results to this filename requires --export to be set as well Example --export- - filename=backtest_today.json (default: backtest- - result.json + filename=user_data/backtest_data/backtest_today.json + (default: user_data/backtest_data/backtest- + result.json) + ``` ### How to use --refresh-pairs-cached parameter? @@ -164,22 +170,29 @@ To optimize your strategy, you can use hyperopt parameter hyperoptimization to find optimal parameter values for your stategy. ``` -usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--enable-position-tracking] - [--timerange TIMERANGE] [-e INT] - [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] +usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] + [--enable-position-stacking] + [--disable-max-market-positions] + [--timerange TIMERANGE] [-e INT] + [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-tracking - Disables buying the same pair multiple times to - simulate real world limitations - --timerange TIMERANGE specify what timerange of data to use. + --enable-position-stacking + Allow buying the same pair twice (position stacking) + --disable-max-market-positions + Disable applying `max_open_trades` during backtest + (same as setting `max_open_trades` to a very high + number) + --timerange TIMERANGE + specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate list. Default: all + ``` ## A parameter missing in the configuration? From 3df79b85429c6f8bddbbcad37cc96c9eab396ecf Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 21:12:05 +0200 Subject: [PATCH 32/47] fix hanging intend --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 3b10b90df..f315ea05b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -188,7 +188,7 @@ class Arguments(object): parser.add_argument( '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' - '(same as setting `max_open_trades` to a very high number)', + '(same as setting `max_open_trades` to a very high number)', action='store_false', dest='use_max_market_positions', default=True From 08237abe20f650a093ab10dad983823ed63ef514 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 09:06:12 +0200 Subject: [PATCH 33/47] Fix wrong backtest duration identified in #1038 --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..1ca2dacec 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -159,7 +159,7 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60, + trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, @@ -174,7 +174,7 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60, + trade_duration=(sell_row.date - buy_row.date)..total_seconds() // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, From 8e4d2abd4e4f522bdfa33c35478c9c12d9a56123 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 09:10:17 +0200 Subject: [PATCH 34/47] Fix typo --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1ca2dacec..0605799b1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -174,7 +174,7 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date)..total_seconds() // 60, + trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, From f9f6a3bd04bc95e7aad3f99db6af805630af86aa Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 09:29:51 +0200 Subject: [PATCH 35/47] cast to int to keep exports constant --- freqtrade/optimize/backtesting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0605799b1..d5039fe7d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -159,7 +159,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, + trade_duration=int(( + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, @@ -174,7 +175,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, + trade_duration=int(( + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, From a374f95687689f414e2fa0475532dcb3b550b7eb Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 18 Jul 2018 14:24:07 +0200 Subject: [PATCH 36/47] Update ccxt from 1.16.50 to 1.16.57 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1ff711df..f22eae180 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.50 +ccxt==1.16.57 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 79b10304359d24dab0eb9382692640072269a573 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 20:08:55 +0200 Subject: [PATCH 37/47] output duration in a more readable way --- freqtrade/optimize/backtesting.py | 10 ++++++---- freqtrade/tests/optimize/test_backtesting.py | 9 ++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d5039fe7d..d60ce9a54 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,7 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace -from datetime import datetime +from datetime import datetime, timedelta from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow @@ -88,7 +88,7 @@ class Backtesting(object): """ stake_currency = str(self.config.get('stake_currency')) - floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f') + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f', '.1f', '.1f') tabular_data = [] headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] @@ -100,7 +100,8 @@ class Backtesting(object): result.profit_percent.mean() * 100.0, result.profit_percent.sum() * 100.0, result.profit_abs.sum(), - result.trade_duration.mean(), + str(timedelta( + minutes=round(result.trade_duration.mean()))) if len(result) else 'nan', len(result[result.profit_abs > 0]), len(result[result.profit_abs < 0]) ]) @@ -112,7 +113,8 @@ class Backtesting(object): results.profit_percent.mean() * 100.0, results.profit_percent.sum() * 100.0, results.profit_abs.sum(), - results.trade_duration.mean(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if len(results) else 'nan', len(results[results.profit_abs > 0]), len(results[results.profit_abs < 0]) ]) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6fbf71e40..103137818 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -392,15 +392,14 @@ def test_generate_text_table(default_conf, mocker): result_str = ( '| pair | buy count | avg profit % | cum profit % | ' - 'total profit BTC | avg duration | profit | loss |\n' + 'total profit BTC | avg duration | profit | loss |\n' '|:--------|------------:|---------------:|---------------:|' - '-------------------:|---------------:|---------:|-------:|\n' + '-------------------:|:---------------|---------:|-------:|\n' '| ETH/BTC | 2 | 15.00 | 30.00 | ' - '0.60000000 | 20.0 | 2 | 0 |\n' + '0.60000000 | 0:20:00 | 2 | 0 |\n' '| TOTAL | 2 | 15.00 | 30.00 | ' - '0.60000000 | 20.0 | 2 | 0 |' + '0.60000000 | 0:20:00 | 2 | 0 |' ) - print(result_str) assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str From 789b98015fb0fd7a586e5ac316df521e49e1802a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 22:52:57 +0200 Subject: [PATCH 38/47] Allow different loglevels --- freqtrade/arguments.py | 8 +++----- freqtrade/configuration.py | 17 +++++++++++++++-- freqtrade/main.py | 12 +----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 731c5d88c..b43755c6f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -3,7 +3,6 @@ This module contains the argument manager class """ import argparse -import logging import os import re from typing import List, NamedTuple, Optional @@ -64,11 +63,10 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='be verbose', - action='store_const', + help='verbose mode (-vv for more, -vvv to get all messages)', + action='count', dest='loglevel', - const=logging.DEBUG, - default=logging.INFO, + default=0, ) self.parser.add_argument( '--version', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 582b2889c..7e3cf3e69 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,10 +12,22 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants - logger = logging.getLogger(__name__) +def set_loggers(log_level: int = 0) -> None: + """ + Set the logger level for Third party libs + :return: None + """ + + logging.getLogger('requests').setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel( + logging.INFO if log_level <= 2 else logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.INFO) + + class Configuration(object): """ Class to read and init the bot configuration @@ -81,9 +93,10 @@ class Configuration(object): if 'loglevel' in self.args and self.args.loglevel: config.update({'loglevel': self.args.loglevel}) logging.basicConfig( - level=config['loglevel'], + level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', ) + set_loggers(config['loglevel']) logger.info('Log level set to %s', logging.getLevelName(config['loglevel'])) # Add dynamic_whitelist if found diff --git a/freqtrade/main.py b/freqtrade/main.py index 977212faf..3ed478ec3 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,7 @@ from typing import List from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State from freqtrade.rpc import RPCMessageType @@ -84,16 +84,6 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: return freqtrade -def set_loggers() -> None: - """ - Set the logger level for Third party libs - :return: None - """ - logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) - logging.getLogger('ccxt.base.exchange').setLevel(logging.INFO) - logging.getLogger('telegram').setLevel(logging.INFO) - - if __name__ == '__main__': set_loggers() main(sys.argv[1:]) From 1ab7f5fb6d4cfbe445c05d7ded427ae539e0d715 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 22:53:44 +0200 Subject: [PATCH 39/47] add tests for more debug levels --- freqtrade/tests/test_arguments.py | 11 ++++---- freqtrade/tests/test_configuration.py | 38 ++++++++++++++++++++++++++- freqtrade/tests/test_main.py | 25 +++--------------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 8a41e3379..07018c79e 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -5,7 +5,6 @@ Unit test file for arguments.py """ import argparse -import logging import pytest @@ -35,7 +34,7 @@ def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' assert args.dynamic_whitelist is None - assert args.loglevel == logging.INFO + assert args.loglevel == 0 def test_parse_args_config() -> None: @@ -53,10 +52,10 @@ def test_parse_args_db_url() -> None: def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 args = Arguments(['--verbose'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 def test_scripts_options() -> None: @@ -153,7 +152,7 @@ def test_parse_args_backtesting_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval == '1m' @@ -170,7 +169,7 @@ def test_parse_args_hyperopt_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.epochs == 20 - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.spaces == ['buy'] assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e64e1b486..5fa8e2184 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,6 +6,7 @@ Unit test file for configuration.py import json from argparse import Namespace from copy import deepcopy +import logging from unittest.mock import MagicMock import pytest @@ -13,7 +14,7 @@ from jsonschema import ValidationError from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has @@ -402,3 +403,38 @@ def test_check_exchange(default_conf) -> None: match=r'.*Exchange "unknown_exchange" not supported.*' ): configuration.check_exchange(conf) + + +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + previous_value1 = logging.getLogger('requests').level + previous_value2 = logging.getLogger('ccxt.base.exchange').level + previous_value3 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('ccxt.base.exchange').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + value3 = logging.getLogger('telegram').level + assert previous_value3 is not value3 + assert value3 is logging.INFO + + set_loggers(log_level=2) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.INFO + assert logging.getLogger('telegram').level is logging.INFO + + set_loggers(log_level=3) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG + assert logging.getLogger('telegram').level is logging.INFO diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 20a02eedc..446945a07 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,7 +2,6 @@ Unit test file for main.py """ -import logging from copy import deepcopy from unittest.mock import MagicMock @@ -11,7 +10,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, reconfigure, set_loggers +from freqtrade.main import main, reconfigure from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange @@ -27,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None: call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' assert call_args.live is False - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval is None @@ -42,29 +41,11 @@ def test_main_start_hyperopt(mocker) -> None: assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.func is not None -def test_set_loggers() -> None: - """ - Test set_loggers() update the logger level for third-party libraries - """ - previous_value1 = logging.getLogger('requests.packages.urllib3').level - previous_value2 = logging.getLogger('telegram').level - - set_loggers() - - value1 = logging.getLogger('requests.packages.urllib3').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('telegram').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - def test_main_fatal_exception(mocker, default_conf, caplog) -> None: """ Test main() function From 75c0a476f8e1c9e7a749fd6f8331398c11921eaa Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 23:36:25 +0200 Subject: [PATCH 40/47] Test setting verbosity in commandline --- freqtrade/tests/test_configuration.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 5fa8e2184..3e088195f 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -405,6 +405,25 @@ def test_check_exchange(default_conf) -> None: configuration.check_exchange(conf) +def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: + """ + Test Configuration.load_config() with cli params used + """ + + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf))) + # Prevent setting loggers + mocker.patch('freqtrade.configuration.set_loggers', MagicMock) + arglist = ['-vvv'] + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf.get('loglevel') == 3 + assert log_has('Log level set to Level 3', caplog.record_tuples) + + def test_set_loggers() -> None: """ Test set_loggers() update the logger level for third-party libraries From aa69177436c42b09237a60199116c105931c7454 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 13:14:21 +0200 Subject: [PATCH 41/47] Properly check emptyness and adjust floatfmt --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d60ce9a54..83b543136 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -88,7 +88,7 @@ class Backtesting(object): """ stake_currency = str(self.config.get('stake_currency')) - floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f', '.1f', '.1f') + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') tabular_data = [] headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] @@ -101,7 +101,7 @@ class Backtesting(object): result.profit_percent.sum() * 100.0, result.profit_abs.sum(), str(timedelta( - minutes=round(result.trade_duration.mean()))) if len(result) else 'nan', + minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', len(result[result.profit_abs > 0]), len(result[result.profit_abs < 0]) ]) @@ -114,7 +114,7 @@ class Backtesting(object): results.profit_percent.sum() * 100.0, results.profit_abs.sum(), str(timedelta( - minutes=round(results.trade_duration.mean()))) if len(results) else 'nan', + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', len(results[results.profit_abs > 0]), len(results[results.profit_abs < 0]) ]) From 8f254031c655fa297aef4eb782aea790d6ba0a44 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 13:19:36 +0200 Subject: [PATCH 42/47] Add short form for parameters, change default for hyperopt --- freqtrade/arguments.py | 6 +++--- freqtrade/optimize/hyperopt.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index f315ea05b..16a01396b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -178,15 +178,15 @@ class Arguments(object): type=str, ) parser.add_argument( - '--enable-position-stacking', - help='Allow buying the same pair twice (position stacking)', + '--eps', '--enable-position-stacking', + help='Allow buying the same pair multiple times (position stacking)', action='store_true', dest='position_stacking', default=False ) parser.add_argument( - '--disable-max-market-positions', + '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' '(same as setting `max_open_trades` to a very high number)', action='store_false', diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 73b74e6ad..addfe19ee 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -280,7 +280,7 @@ class Hyperopt(Backtesting): { 'stake_amount': self.config['stake_amount'], 'processed': processed, - 'position_stacking': self.config.get('position_stacking', False), + 'position_stacking': self.config.get('position_stacking', True), } ) result_explanation = self.format_results(results) From 71100a67c952e9910320a1805318ce283ac6dd5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 13:20:15 +0200 Subject: [PATCH 43/47] update documentation with new options --- docs/bot-usage.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index d5c76f4a4..4e479adac 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -117,9 +117,7 @@ python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.s Backtesting also uses the config specified via `-c/--config`. ``` -usage: main.py backtesting [-h] [-i TICKER_INTERVAL] - [--enable-position-stacking] - [--disable-max-market-positions] +usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] [--export-filename PATH] @@ -127,9 +125,10 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-stacking - Allow buying the same pair twice (position stacking) - --disable-max-market-positions + --eps, --enable-position-stacking + Allow buying the same pair multiple times (position + stacking) + --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number) @@ -170,9 +169,7 @@ To optimize your strategy, you can use hyperopt parameter hyperoptimization to find optimal parameter values for your stategy. ``` -usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] - [--enable-position-stacking] - [--disable-max-market-positions] +usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] [--timerange TIMERANGE] [-e INT] [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] @@ -180,9 +177,10 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-stacking - Allow buying the same pair twice (position stacking) - --disable-max-market-positions + --eps, --enable-position-stacking + Allow buying the same pair multiple times (position + stacking) + --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number) From f2bfc9ccc2939fa262b4812a98f7f8e8a45c3263 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 19 Jul 2018 14:24:07 +0200 Subject: [PATCH 44/47] Update ccxt from 1.16.57 to 1.16.68 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f22eae180..576185f06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.57 +ccxt==1.16.68 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 90915b6b2f794f4d1e635b1aa0ae9d94a006dbb6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 20:43:41 +0200 Subject: [PATCH 45/47] Revert "Add more verbosity levels" --- freqtrade/arguments.py | 8 ++-- freqtrade/configuration.py | 17 +------- freqtrade/main.py | 12 +++++- freqtrade/tests/test_arguments.py | 11 +++--- freqtrade/tests/test_configuration.py | 57 +-------------------------- freqtrade/tests/test_main.py | 25 ++++++++++-- 6 files changed, 47 insertions(+), 83 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 022a2c739..16a01396b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -3,6 +3,7 @@ This module contains the argument manager class """ import argparse +import logging import os import re from typing import List, NamedTuple, Optional @@ -63,10 +64,11 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='verbose mode (-vv for more, -vvv to get all messages)', - action='count', + help='be verbose', + action='store_const', dest='loglevel', - default=0, + const=logging.DEBUG, + default=logging.INFO, ) self.parser.add_argument( '--version', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 76faa9819..f5c1c398d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,22 +12,10 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants + logger = logging.getLogger(__name__) -def set_loggers(log_level: int = 0) -> None: - """ - Set the logger level for Third party libs - :return: None - """ - - logging.getLogger('requests').setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) - logging.getLogger("urllib3").setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) - logging.getLogger('ccxt.base.exchange').setLevel( - logging.INFO if log_level <= 2 else logging.DEBUG) - logging.getLogger('telegram').setLevel(logging.INFO) - - class Configuration(object): """ Class to read and init the bot configuration @@ -93,10 +81,9 @@ class Configuration(object): if 'loglevel' in self.args and self.args.loglevel: config.update({'loglevel': self.args.loglevel}) logging.basicConfig( - level=logging.DEBUG, + level=config['loglevel'], format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', ) - set_loggers(config['loglevel']) logger.info('Log level set to %s', logging.getLevelName(config['loglevel'])) # Add dynamic_whitelist if found diff --git a/freqtrade/main.py b/freqtrade/main.py index 3ed478ec3..977212faf 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,7 @@ from typing import List from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration, set_loggers +from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State from freqtrade.rpc import RPCMessageType @@ -84,6 +84,16 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: return freqtrade +def set_loggers() -> None: + """ + Set the logger level for Third party libs + :return: None + """ + logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) + logging.getLogger('ccxt.base.exchange').setLevel(logging.INFO) + logging.getLogger('telegram').setLevel(logging.INFO) + + if __name__ == '__main__': set_loggers() main(sys.argv[1:]) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 07018c79e..8a41e3379 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -5,6 +5,7 @@ Unit test file for arguments.py """ import argparse +import logging import pytest @@ -34,7 +35,7 @@ def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' assert args.dynamic_whitelist is None - assert args.loglevel == 0 + assert args.loglevel == logging.INFO def test_parse_args_config() -> None: @@ -52,10 +53,10 @@ def test_parse_args_db_url() -> None: def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() - assert args.loglevel == 1 + assert args.loglevel == logging.DEBUG args = Arguments(['--verbose'], '').get_parsed_arg() - assert args.loglevel == 1 + assert args.loglevel == logging.DEBUG def test_scripts_options() -> None: @@ -152,7 +153,7 @@ def test_parse_args_backtesting_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True - assert call_args.loglevel == 0 + assert call_args.loglevel == logging.INFO assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval == '1m' @@ -169,7 +170,7 @@ def test_parse_args_hyperopt_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.epochs == 20 - assert call_args.loglevel == 0 + assert call_args.loglevel == logging.INFO assert call_args.subparser == 'hyperopt' assert call_args.spaces == ['buy'] assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e4ef9d8f5..da8d3bebf 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,7 +6,6 @@ Unit test file for configuration.py import json from argparse import Namespace from copy import deepcopy -import logging from unittest.mock import MagicMock import pytest @@ -14,7 +13,7 @@ from jsonschema import ValidationError from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration, set_loggers +from freqtrade.configuration import Configuration from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has @@ -407,57 +406,3 @@ def test_check_exchange(default_conf) -> None: match=r'.*Exchange "unknown_exchange" not supported.*' ): configuration.check_exchange(conf) - - -def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: - """ - Test Configuration.load_config() with cli params used - """ - - mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(default_conf))) - # Prevent setting loggers - mocker.patch('freqtrade.configuration.set_loggers', MagicMock) - arglist = ['-vvv'] - args = Arguments(arglist, '').get_parsed_arg() - - configuration = Configuration(args) - validated_conf = configuration.load_config() - - assert validated_conf.get('loglevel') == 3 - assert log_has('Log level set to Level 3', caplog.record_tuples) - - -def test_set_loggers() -> None: - """ - Test set_loggers() update the logger level for third-party libraries - """ - previous_value1 = logging.getLogger('requests').level - previous_value2 = logging.getLogger('ccxt.base.exchange').level - previous_value3 = logging.getLogger('telegram').level - - set_loggers() - - value1 = logging.getLogger('requests').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('ccxt.base.exchange').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - value3 = logging.getLogger('telegram').level - assert previous_value3 is not value3 - assert value3 is logging.INFO - - set_loggers(log_level=2) - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.INFO - assert logging.getLogger('telegram').level is logging.INFO - - set_loggers(log_level=3) - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG - assert logging.getLogger('telegram').level is logging.INFO diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 446945a07..20a02eedc 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,6 +2,7 @@ Unit test file for main.py """ +import logging from copy import deepcopy from unittest.mock import MagicMock @@ -10,7 +11,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, reconfigure +from freqtrade.main import main, reconfigure, set_loggers from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange @@ -26,7 +27,7 @@ def test_parse_args_backtesting(mocker) -> None: call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' assert call_args.live is False - assert call_args.loglevel == 0 + assert call_args.loglevel == 20 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval is None @@ -41,11 +42,29 @@ def test_main_start_hyperopt(mocker) -> None: assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' - assert call_args.loglevel == 0 + assert call_args.loglevel == 20 assert call_args.subparser == 'hyperopt' assert call_args.func is not None +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + previous_value1 = logging.getLogger('requests.packages.urllib3').level + previous_value2 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests.packages.urllib3').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('telegram').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + def test_main_fatal_exception(mocker, default_conf, caplog) -> None: """ Test main() function From dd1290e38e3f01cccf1959ea5155a30725a73814 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 21:12:27 +0200 Subject: [PATCH 46/47] Add multiple verbosity levels --- freqtrade/arguments.py | 8 ++-- freqtrade/configuration.py | 29 +++++++++--- freqtrade/main.py | 12 +---- freqtrade/tests/test_arguments.py | 11 +++-- freqtrade/tests/test_configuration.py | 63 ++++++++++++++++++++++++++- freqtrade/tests/test_main.py | 25 ++--------- 6 files changed, 96 insertions(+), 52 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 16a01396b..022a2c739 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -3,7 +3,6 @@ This module contains the argument manager class """ import argparse -import logging import os import re from typing import List, NamedTuple, Optional @@ -64,11 +63,10 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='be verbose', - action='store_const', + help='verbose mode (-vv for more, -vvv to get all messages)', + action='count', dest='loglevel', - const=logging.DEBUG, - default=logging.INFO, + default=0, ) self.parser.add_argument( '--version', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index f5c1c398d..dcc6e4332 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,10 +12,22 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants - logger = logging.getLogger(__name__) +def set_loggers(log_level: int = 0) -> None: + """ + Set the logger level for Third party libs + :return: None + """ + + logging.getLogger('requests').setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel( + logging.INFO if log_level <= 2 else logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.INFO) + + class Configuration(object): """ Class to read and init the bot configuration @@ -79,12 +91,15 @@ class Configuration(object): # Log level if 'loglevel' in self.args and self.args.loglevel: - config.update({'loglevel': self.args.loglevel}) - logging.basicConfig( - level=config['loglevel'], - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) - logger.info('Log level set to %s', logging.getLevelName(config['loglevel'])) + config.update({'verbosity': self.args.loglevel}) + else: + config.update({'verbosity': 0}) + logging.basicConfig( + level=logging.INFO if config['verbosity'] < 1 else logging.DEBUG, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + set_loggers(config['verbosity']) + logger.info('Verbosity set to %s', config['verbosity']) # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: diff --git a/freqtrade/main.py b/freqtrade/main.py index 977212faf..3ed478ec3 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,7 @@ from typing import List from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State from freqtrade.rpc import RPCMessageType @@ -84,16 +84,6 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: return freqtrade -def set_loggers() -> None: - """ - Set the logger level for Third party libs - :return: None - """ - logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) - logging.getLogger('ccxt.base.exchange').setLevel(logging.INFO) - logging.getLogger('telegram').setLevel(logging.INFO) - - if __name__ == '__main__': set_loggers() main(sys.argv[1:]) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 8a41e3379..07018c79e 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -5,7 +5,6 @@ Unit test file for arguments.py """ import argparse -import logging import pytest @@ -35,7 +34,7 @@ def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' assert args.dynamic_whitelist is None - assert args.loglevel == logging.INFO + assert args.loglevel == 0 def test_parse_args_config() -> None: @@ -53,10 +52,10 @@ def test_parse_args_db_url() -> None: def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 args = Arguments(['--verbose'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 def test_scripts_options() -> None: @@ -153,7 +152,7 @@ def test_parse_args_backtesting_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval == '1m' @@ -170,7 +169,7 @@ def test_parse_args_hyperopt_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.epochs == 20 - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.spaces == ['buy'] assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index da8d3bebf..a8a2c5fce 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,6 +6,7 @@ Unit test file for configuration.py import json from argparse import Namespace from copy import deepcopy +import logging from unittest.mock import MagicMock import pytest @@ -13,7 +14,7 @@ from jsonschema import ValidationError from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has @@ -406,3 +407,63 @@ def test_check_exchange(default_conf) -> None: match=r'.*Exchange "unknown_exchange" not supported.*' ): configuration.check_exchange(conf) + + +def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: + """ + Test Configuration.load_config() with cli params used + """ + + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf))) + # Prevent setting loggers + mocker.patch('freqtrade.configuration.set_loggers', MagicMock) + arglist = ['-vvv'] + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf.get('verbosity') == 3 + assert log_has('Verbosity set to 3', caplog.record_tuples) + + +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + # Reset Logging to Debug, otherwise this fails randomly as it's set globally + logging.getLogger('requests').setLevel(logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel(logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.DEBUG) + + previous_value1 = logging.getLogger('requests').level + previous_value2 = logging.getLogger('ccxt.base.exchange').level + previous_value3 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('ccxt.base.exchange').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + value3 = logging.getLogger('telegram').level + assert previous_value3 is not value3 + assert value3 is logging.INFO + + set_loggers(log_level=2) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.INFO + assert logging.getLogger('telegram').level is logging.INFO + + set_loggers(log_level=3) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG + assert logging.getLogger('telegram').level is logging.INFO diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 20a02eedc..446945a07 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,7 +2,6 @@ Unit test file for main.py """ -import logging from copy import deepcopy from unittest.mock import MagicMock @@ -11,7 +10,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, reconfigure, set_loggers +from freqtrade.main import main, reconfigure from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange @@ -27,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None: call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' assert call_args.live is False - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval is None @@ -42,29 +41,11 @@ def test_main_start_hyperopt(mocker) -> None: assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.func is not None -def test_set_loggers() -> None: - """ - Test set_loggers() update the logger level for third-party libraries - """ - previous_value1 = logging.getLogger('requests.packages.urllib3').level - previous_value2 = logging.getLogger('telegram').level - - set_loggers() - - value1 = logging.getLogger('requests.packages.urllib3').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('telegram').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - def test_main_fatal_exception(mocker, default_conf, caplog) -> None: """ Test main() function From 7f6c79eb76c3c90f986edcd31975b9153cb7ba6a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 20 Jul 2018 14:24:06 +0200 Subject: [PATCH 47/47] Update ccxt from 1.16.68 to 1.16.75 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 576185f06..e54952c23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.68 +ccxt==1.16.75 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1