From 20ebd744c38cfb0791b4176379cda761d11069bb Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 12 May 2018 09:43:22 +0300 Subject: [PATCH 001/239] Freqtrade 0.16.0 release --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index d1671e4c6..cd4515a3b 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.15.1' +__version__ = '0.16.0' class DependencyException(BaseException): From e1322b75a930e13615c3d3ac1c465551369efbd7 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 12 May 2018 09:50:01 +0300 Subject: [PATCH 002/239] Freqtrade 0.16.1 release Note. This is the last release that uses our own bittrex implementation for trading. After this ccxt library will be taken into use which will offer the needed exchanges (bittrex/binance) --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index cd4515a3b..ae8e57fd5 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.16.0' +__version__ = '0.16.1' class DependencyException(BaseException): From 9be98cd8f77c503f303d021eb62b531c9be698b3 Mon Sep 17 00:00:00 2001 From: Anton Date: Wed, 23 May 2018 13:15:03 +0300 Subject: [PATCH 003/239] Add ability to set unlimited stake_amount --- docs/configuration.md | 2 +- freqtrade/constants.py | 5 +- freqtrade/freqtradebot.py | 25 ++++-- freqtrade/optimize/backtesting.py | 5 +- freqtrade/tests/test_freqtradebot.py | 130 ++++++++++++++++++++++----- 5 files changed, 137 insertions(+), 30 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3d36947c3..e75775f3f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -16,7 +16,7 @@ The table below will list all configuration parameters. |----------|---------|----------|-------------| | `max_open_trades` | 3 | Yes | Number of trades open your bot will have. | `stake_currency` | BTC | Yes | Crypto-currency used for trading. -| `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. +| `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. 'unlimited' is used to allow a bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 7731ea610..20c8a2287 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -32,7 +32,10 @@ CONF_SCHEMA = { 'max_open_trades': {'type': 'integer', 'minimum': 0}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, - 'stake_amount': {'type': 'number', 'minimum': 0.0005}, + 'stake_amount': {'anyOf': [ + {'type': 'integer', 'minimum': 0.0005}, + {'constant': 'unlimited'} + ]}, 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', 'CLP', 'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7c955d423..af3f9287e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -255,6 +255,24 @@ class FreqtradeBot(object): balance = self.config['bid_strategy']['ask_last_balance'] return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + def _get_trade_stake_amount(self) -> float: + stake_amount = self.config['stake_amount'] + avaliable_amount = exchange.get_balance(self.config['stake_currency']) + + if stake_amount == 'unlimited': + open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) + if open_trades == self.config['max_open_trades']: + return 0 + return avaliable_amount / (self.config['max_open_trades'] - open_trades) + + # Check if stake_amount is fulfilled + if avaliable_amount < stake_amount: + raise DependencyException( + 'stake amount is not fulfilled (currency={})'.format(self.config['stake_currency']) + ) + + return stake_amount + def create_trade(self) -> bool: """ Checks the implemented trading indicator(s) for a randomly picked pair, @@ -263,19 +281,14 @@ class FreqtradeBot(object): :param interval: Ticker interval used for Analyze :return: True if a trade object has been created and persisted, False otherwise """ - stake_amount = self.config['stake_amount'] interval = self.analyze.get_ticker_interval() + stake_amount = self._get_trade_stake_amount() logger.info( 'Checking buy signals to create a new trade with stake_amount: %f ...', stake_amount ) whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) - # Check if stake_amount is fulfilled - if exchange.get_balance(self.config['stake_currency']) < stake_amount: - raise DependencyException( - 'stake amount is not fulfilled (currency={})'.format(self.config['stake_currency']) - ) # Remove currently opened and latest pairs from whitelist for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 376730d0f..eb6b25e02 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -13,7 +13,7 @@ from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize -from freqtrade import exchange +from freqtrade import exchange, DependencyException from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -296,6 +296,9 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: config['exchange']['key'] = '' config['exchange']['secret'] = '' + if config['stake_amount'] == 'unlimited': + raise DependencyException('stake amount could not be "unlimited" for backtesting') + return config diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index ebabc0187..d9d009a47 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -220,6 +220,115 @@ def test_refresh_whitelist() -> None: pass +def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: + """ + Test get_trade_stake_amount() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + result = freqtrade._get_trade_stake_amount() + assert(result == conf['stake_amount']) + + +def test_get_trade_stake_amount_no_stake_amount(default_conf, + ticker, + limit_buy_order, + fee, + mocker) -> None: + """ + Test get_trade_stake_amount() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + with pytest.raises(DependencyException, match=r'.*stake amount.*'): + freqtrade._get_trade_stake_amount() + + +def test_get_trade_stake_amount_unlimited_amount(default_conf, + ticker, + limit_buy_order, + fee, + mocker) -> None: + """ + Test get_trade_stake_amount() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_balance=MagicMock(return_value=default_conf['stake_amount']), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['stake_amount'] = 'unlimited' + conf['max_open_trades'] = 2 + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + # no open trades, order amount should be 'balance / max_open_trades' + result = freqtrade._get_trade_stake_amount() + assert(result == default_conf['stake_amount'] / conf['max_open_trades']) + + # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' + freqtrade.create_trade() + + result = freqtrade._get_trade_stake_amount() + assert(result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1)) + + +def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), + get_fee=fee, + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + with pytest.raises(DependencyException, match=r'.*stake amount.*'): + freqtrade.create_trade() + + def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> None: """ Test create_trade() method @@ -281,27 +390,6 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, assert rate * amount >= conf['stake_amount'] -def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: - """ - Test create_trade() method - """ - patch_get_signal(mocker) - patch_RPCManager(mocker) - patch_coinmarketcap(mocker) - mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), - get_fee=fee, - ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) - - with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade.create_trade() - - def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocker) -> None: """ Test create_trade() method From cf5d691950c99c8c055048e1b5aa4150b136a765 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 25 May 2018 00:46:08 +0300 Subject: [PATCH 004/239] Clean the tests --- freqtrade/tests/test_freqtradebot.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index d9d009a47..e47bfd007 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -224,23 +224,18 @@ def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mock """ Test get_trade_stake_amount() method """ - patch_get_signal(mocker) + patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - get_fee=fee, + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2) ) - conf = deepcopy(default_conf) - - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) result = freqtrade._get_trade_stake_amount() - assert(result == conf['stake_amount']) + assert(result == default_conf['stake_amount']) def test_get_trade_stake_amount_no_stake_amount(default_conf, @@ -251,21 +246,14 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, """ Test get_trade_stake_amount() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), - get_fee=fee, + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) ) - conf = deepcopy(default_conf) - - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade._get_trade_stake_amount() From 3427c7eb540ac7b73e7ed52578ec1c281b4b2500 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 25 May 2018 17:04:08 +0300 Subject: [PATCH 005/239] Use constants --- docs/configuration.md | 2 +- freqtrade/constants.py | 3 ++- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 7 ++++--- freqtrade/tests/test_freqtradebot.py | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e75775f3f..5b527ddf7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -16,7 +16,7 @@ The table below will list all configuration parameters. |----------|---------|----------|-------------| | `max_open_trades` | 3 | Yes | Number of trades open your bot will have. | `stake_currency` | BTC | Yes | Crypto-currency used for trading. -| `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. 'unlimited' is used to allow a bot to use all avaliable balance. +| `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 20c8a2287..1039b9da4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,6 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +UNLIMITED_STAKE_AMOUNT = 'unlimited' TICKER_INTERVAL_MINUTES = { '1m': 1, @@ -34,7 +35,7 @@ CONF_SCHEMA = { 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, 'stake_amount': {'anyOf': [ {'type': 'integer', 'minimum': 0.0005}, - {'constant': 'unlimited'} + {'constant': UNLIMITED_STAKE_AMOUNT} ]}, 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', 'CLP', 'CNY', 'CZK', 'DKK', diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index af3f9287e..431a146dd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -259,7 +259,7 @@ class FreqtradeBot(object): stake_amount = self.config['stake_amount'] avaliable_amount = exchange.get_balance(self.config['stake_currency']) - if stake_amount == 'unlimited': + if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) if open_trades == self.config['max_open_trades']: return 0 diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index eb6b25e02..520765853 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -13,7 +13,7 @@ from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize -from freqtrade import exchange, DependencyException +from freqtrade import exchange, constants, DependencyException from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -296,8 +296,9 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: config['exchange']['key'] = '' config['exchange']['secret'] = '' - if config['stake_amount'] == 'unlimited': - raise DependencyException('stake amount could not be "unlimited" for backtesting') + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: + raise DependencyException('stake amount could not be "%s" for backtesting' % + constants.UNLIMITED_STAKE_AMOUNT) return config diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e47bfd007..eeab3bf37 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -15,7 +15,7 @@ import pytest import requests from sqlalchemy import create_engine -from freqtrade import DependencyException, OperationalException, TemporaryError +from freqtrade import constants, DependencyException, OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.state import State @@ -280,7 +280,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, ) conf = deepcopy(default_conf) - conf['stake_amount'] = 'unlimited' + conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT conf['max_open_trades'] = 2 freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) From daa9c0c02607cf5db0a860b133ddfbf91fa6dde9 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 4 Jun 2018 01:48:26 +0300 Subject: [PATCH 006/239] Fix review comments --- docs/configuration.md | 7 ++++++ freqtrade/constants.py | 9 +++---- freqtrade/tests/optimize/test_backtesting.py | 25 +++++++++++++++++++- freqtrade/tests/test_configuration.py | 12 ++++++++++ freqtrade/tests/test_freqtradebot.py | 6 +++++ 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5b527ddf7..0bb417338 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -42,6 +42,13 @@ The table below will list all configuration parameters. The definition of each config parameters is in [misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L205). +### Understand stake_amount +`stake_amount` is an amount of crypto-currency your bot will use for each trade. +The minimal value is 0.0005. If there is not enough crypto-currency in +the account an exception is generated. +To allow the bot to trade all the avaliable `stake_currency` in your account set `stake_amount` = `unlimited`. +In this case a trade amount is calclulated as `currency_balanse / (max_open_trades - current_open_trades)`. + ### Understand minimal_roi `minimal_roi` is a JSON object where the key is a duration in minutes and the value is the minimum ROI in percent. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1039b9da4..ea4fc478c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -33,10 +33,11 @@ CONF_SCHEMA = { 'max_open_trades': {'type': 'integer', 'minimum': 0}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, - 'stake_amount': {'anyOf': [ - {'type': 'integer', 'minimum': 0.0005}, - {'constant': UNLIMITED_STAKE_AMOUNT} - ]}, + 'stake_amount': { + "type": ["number", "string"], + "minimum": 0.0005, + "pattern": UNLIMITED_STAKE_AMOUNT + }, 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', 'CLP', 'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f17a0115e..4d02cfe5f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -3,6 +3,7 @@ import json import math import random +import pytest from copy import deepcopy from typing import List from unittest.mock import MagicMock @@ -11,7 +12,7 @@ import numpy as np import pandas as pd from arrow import Arrow -from freqtrade import optimize +from freqtrade import optimize, constants, DependencyException from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration @@ -261,6 +262,28 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non ) +def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + + conf = deepcopy(default_conf) + conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'DefaultStrategy', + 'backtesting' + ] + + with pytest.raises(DependencyException, match=r'.*stake amount.*'): + setup_configuration(get_args(args)) + + def test_start(mocker, fee, default_conf, caplog) -> None: """ Test start() function diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d27405d91..fb63ca2d9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -53,6 +53,18 @@ def test_load_config_missing_attributes(default_conf) -> None: configuration._validate_config(conf) +def test_load_config_incorrect_stake_amount(default_conf) -> None: + """ + Test the configuration validator with a missing attribute + """ + conf = deepcopy(default_conf) + conf['stake_amount'] = 'fake' + + with pytest.raises(ValidationError, match=r'.*\'fake\' does not match \'unlimited\'.*'): + configuration = Configuration([]) + configuration._validate_config(conf) + + def test_load_config_file(default_conf, mocker, caplog) -> None: """ Test Configuration._load_config_file() method diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index eeab3bf37..d91dd4720 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -295,6 +295,12 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, result = freqtrade._get_trade_stake_amount() assert(result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1)) + # create 2 trades, order amount should be 0 + freqtrade.create_trade() + + result = freqtrade._get_trade_stake_amount() + assert(result == 0) + def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: """ From 3030bf9778f4d23d2237432e5c840c6e5c74f374 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 4 Jun 2018 01:52:54 +0300 Subject: [PATCH 007/239] Fix types --- freqtrade/tests/test_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index df4d3f5ef..98adf19e9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -62,7 +62,7 @@ def test_load_config_incorrect_stake_amount(default_conf) -> None: conf['stake_amount'] = 'fake' with pytest.raises(ValidationError, match=r'.*\'fake\' does not match \'unlimited\'.*'): - configuration = Configuration([]) + configuration = Configuration(Namespace()) configuration._validate_config(conf) From 12d8a8b1a35b463fc90a0af0c9ca6d1d0560538e Mon Sep 17 00:00:00 2001 From: Anton Date: Wed, 6 Jun 2018 00:14:28 +0300 Subject: [PATCH 008/239] Fix review comments --- freqtrade/arguments.py | 10 ++++------ freqtrade/freqtradebot.py | 6 ++++-- freqtrade/tests/test_freqtradebot.py | 6 ++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 97c3d8cb2..1ae805ca5 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -239,17 +239,15 @@ class Arguments(object): stop: Optional[int] = None if stype[0]: starts = rvals[index] - if stype[0] == 'date': - start = int(starts) if len(starts) == 10 \ - else arrow.get(starts, 'YYYYMMDD').timestamp + if stype[0] == 'date' and len(starts) == 8: + start = arrow.get(starts, 'YYYYMMDD').timestamp else: start = int(starts) index += 1 if stype[1]: stops = rvals[index] - if stype[1] == 'date': - stop = int(stops) if len(stops) == 10 \ - else arrow.get(stops, 'YYYYMMDD').timestamp + if stype[1] == 'date' and len(stops) == 8: + stop = arrow.get(stops, 'YYYYMMDD').timestamp else: stop = int(stops) return stype, start, stop diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b1928a4d5..49c179907 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -258,14 +258,16 @@ class FreqtradeBot(object): if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) - if open_trades == self.config['max_open_trades']: + if open_trades >= self.config['max_open_trades']: return 0 return avaliable_amount / (self.config['max_open_trades'] - open_trades) # Check if stake_amount is fulfilled if avaliable_amount < stake_amount: raise DependencyException( - 'stake amount is not fulfilled (currency={})'.format(self.config['stake_currency']) + 'Available balance(%f %s) is lower than stake amount(%f %s)' % ( + avaliable_amount, self.config['stake_currency'], + stake_amount, self.config['stake_currency']) ) return stake_amount diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index d91dd4720..581e76406 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -301,6 +301,12 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, result = freqtrade._get_trade_stake_amount() assert(result == 0) + # set max_open_trades = 0, so do not trade + conf['max_open_trades'] = 0 + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + result = freqtrade._get_trade_stake_amount() + assert(result == 0) + def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: """ From b1b87731b1a97de0fd8d2cd262c22c80fed87f30 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 8 Jun 2018 00:54:46 +0300 Subject: [PATCH 009/239] Support case when _get_trade_stake_amount returns None --- freqtrade/freqtradebot.py | 6 ++++- freqtrade/tests/test_freqtradebot.py | 36 +++++++++++++++------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9392f5b39..70c74755e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -257,7 +257,8 @@ class FreqtradeBot(object): if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) if open_trades >= self.config['max_open_trades']: - return 0 + logger.warning('Can\'t open a new trade: max number of trades is reached') + return None return avaliable_amount / (self.config['max_open_trades'] - open_trades) # Check if stake_amount is fulfilled @@ -281,6 +282,9 @@ class FreqtradeBot(object): interval = self.analyze.get_ticker_interval() stake_amount = self._get_trade_stake_amount() + if not stake_amount: + return False + logger.info( 'Checking buy signals to create a new trade with stake_amount: %f ...', stake_amount diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index f7e84ebdd..a90b1f983 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -231,7 +231,7 @@ def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mock get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2) ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) result = freqtrade._get_trade_stake_amount() assert(result == default_conf['stake_amount']) @@ -252,7 +252,7 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade._get_trade_stake_amount() @@ -282,29 +282,29 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT conf['max_open_trades'] = 2 - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) # no open trades, order amount should be 'balance / max_open_trades' result = freqtrade._get_trade_stake_amount() - assert(result == default_conf['stake_amount'] / conf['max_open_trades']) + assert result == default_conf['stake_amount'] / conf['max_open_trades'] # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' freqtrade.create_trade() result = freqtrade._get_trade_stake_amount() - assert(result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1)) + assert result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1) - # create 2 trades, order amount should be 0 + # create 2 trades, order amount should be None freqtrade.create_trade() result = freqtrade._get_trade_stake_amount() - assert(result == 0) + assert result is None - # set max_open_trades = 0, so do not trade + # set max_open_trades = None, so do not trade conf['max_open_trades'] = 0 - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) result = freqtrade._get_trade_stake_amount() - assert(result == 0) + assert result is None def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: @@ -322,7 +322,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade.create_trade() @@ -389,7 +389,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, assert rate * amount >= conf['stake_amount'] -def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, fee, mocker) -> None: """ Test create_trade() method """ @@ -401,13 +401,17 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), + get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + conf = deepcopy(default_conf) + conf['max_open_trades'] = 0 + conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT - with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade.create_trade() + freqtrade = FreqtradeBot(conf) + + assert freqtrade.create_trade() is False + assert freqtrade._get_trade_stake_amount() is None def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocker) -> None: From f0456bb802257af9ddbabe088142b566908be00e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 5 Jun 2018 00:22:44 -0700 Subject: [PATCH 010/239] Update the README structure --- README.md | 169 ++++++++++++++++++++----------------------- docs/installation.md | 23 ++++++ 2 files changed, 102 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 24e01531c..94ebb7bf8 100644 --- a/README.md +++ b/README.md @@ -22,33 +22,10 @@ expect. We strongly recommend you to have coding and Python knowledge. Do not hesitate to read the source code and understand the mechanism of this bot. -## Table of Contents -- [Features](#features) -- [Quick start](#quick-start) -- [Documentations](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) - - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) - - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) -- [Support](#support) - - [Help](#help--slack) - - [Bugs](#bugs--issues) - - [Feature Requests](#feature-requests) - - [Pull Requests](#pull-requests) -- [Basic Usage](#basic-usage) - - [Bot commands](#bot-commands) - - [Telegram RPC commands](#telegram-rpc-commands) -- [Requirements](#requirements) - - [Min hardware required](#min-hardware-required) - - [Software requirements](#software-requirements) - -## Branches -The project is currently setup in two main branches: -- `develop` - This branch has often new features, but might also cause -breaking changes. -- `master` - This branch contains the latest stable release. The bot -'should' be stable on this branch, and is generally well tested. +## Exchange marketplaces supported +- [X] [Bittrex](https://bittrex.com/) +- [X] [Binance](https://www.binance.com/) +- [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Features - [x] **Based on Python 3.6+**: For botting on any operating system - @@ -65,74 +42,50 @@ strategy parameters with real exchange data. - [x] **Daily summary of profit/loss**: Provide a daily summary of your profit/loss. - [x] **Performance status report**: Provide a performance status of your current trades. -### Exchange marketplaces supported -- [X] [Bittrex](https://bittrex.com/) -- [X] [Binance](https://www.binance.com/) -- [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +## Table of Contents +- [Quick start](#quick-start) +- [Documentations](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) + - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) + - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) + - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) + - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) +- [Basic Usage](#basic-usage) + - [Bot commands](#bot-commands) + - [Telegram RPC commands](#telegram-rpc-commands) +- [Support](#support) + - [Help](#help--slack) + - [Bugs](#bugs--issues) + - [Feature Requests](#feature-requests) + - [Pull Requests](#pull-requests) +- [Requirements](#requirements) + - [Min hardware required](#min-hardware-required) + - [Software requirements](#software-requirements) ## Quick start -This quick start section is a very short explanation on how to test the -bot in dry-run. We invite you to read the -[bot documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) -to ensure you understand how the bot is working. - -### Easy installation -The script below will install all dependencies and help you to configure the bot. -```bash -./setup.sh --install -``` - -### Manual installation -The following steps are made for Linux/MacOS environment - -**1. Clone the repo** +Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. ```bash git clone git@github.com:freqtrade/freqtrade.git git checkout develop cd freqtrade +./setup.sh --install ``` -**2. Create the config file** -Switch `"dry_run": true,` -```bash -cp config.json.example config.json -vi config.json -``` -**3. Build your docker image and run it** -```bash -docker build -t freqtrade . -docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` +_Windows installation is explained in [Installation doc](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md)_ -### Help / Slack -For any questions not covered by the documentation or for further -information about the bot, we encourage you to join our slack channel. -- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). +## Documentation +We invite you to read the bot documentation to ensure you understand how the bot is working. +- [Index](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) +- [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) +- [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) +- [Bot usage](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md) + - [How to run the bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) + - [How to use Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) + - [How to use Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) +- [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) +- [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) +- [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) -### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) -If you discover a bug in the bot, please -[search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) -first. If it hasn't been reported, please -[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and -ensure you follow the template guide so that our team can assist you as -quickly as possible. - -### [Feature Requests](https://github.com/freqtrade/freqtrade/labels/enhancement) -Have you a great idea to improve the bot you want to share? Please, -first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement). -If it hasn't been requested, please -[create a new request](https://github.com/freqtrade/freqtrade/issues/new) -and ensure you follow the template guide so that it does not get lost -in the bug reports. - -### [Pull Requests](https://github.com/freqtrade/freqtrade/pulls) -Feel like our bot is missing a feature? We welcome your pull requests! -Please read our -[Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) -to understand the requirements before sending your pull-requests. - -**Important:** Always create your PR against the `develop` branch, not -`master`. ## Basic Usage @@ -170,11 +123,7 @@ optional arguments: "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is enabled. ``` -More details on: -- [How to run the bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) -- [How to use Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) -- [How to use Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) - + ### Telegram RPC commands Telegram is not mandatory. However, this is a great way to control your bot. More details on our @@ -193,6 +142,46 @@ bot. More details on our - `/help`: Show help message - `/version`: Show version + +## Development branches +The project is currently setup in two main branches: +- `develop` - This branch has often new features, but might also cause +breaking changes. +- `master` - This branch contains the latest stable release. The bot +'should' be stable on this branch, and is generally well tested. + + +## Support +### Help / Slack +For any questions not covered by the documentation or for further +information about the bot, we encourage you to join our slack channel. +- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). + +### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) +If you discover a bug in the bot, please +[search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) +first. If it hasn't been reported, please +[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and +ensure you follow the template guide so that our team can assist you as +quickly as possible. + +### [Feature Requests](https://github.com/freqtrade/freqtrade/labels/enhancement) +Have you a great idea to improve the bot you want to share? Please, +first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement). +If it hasn't been requested, please +[create a new request](https://github.com/freqtrade/freqtrade/issues/new) +and ensure you follow the template guide so that it does not get lost +in the bug reports. + +### [Pull Requests](https://github.com/freqtrade/freqtrade/pulls) +Feel like our bot is missing a feature? We welcome your pull requests! +Please read our +[Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) +to understand the requirements before sending your pull-requests. + +**Important:** Always create your PR against the `develop` branch, not +`master`. + ## Requirements ### Min hardware required diff --git a/docs/installation.md b/docs/installation.md index 9818529f6..9fec6d25e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,6 +8,7 @@ To understand how to set up the bot please read the [Bot Configuration](https:// * [Table of Contents](#table-of-contents) * [Easy Installation - Linux Script](#easy-installation---linux-script) +* [Manual installation](#manual-installation) * [Automatic Installation - Docker](#automatic-installation---docker) * [Custom Linux MacOS Installation](#custom-installation) - [Requirements](#requirements) @@ -55,6 +56,28 @@ Reset parameter will hard reset your branch (only if you are on `master` or `dev Config parameter is a `config.json` configurator. This script will ask you questions to setup your bot and create your `config.json`. + +## Manual installation - Linux/MacOS +The following steps are made for Linux/MacOS environment + +**1. Clone the repo** +```bash +git clone git@github.com:freqtrade/freqtrade.git +git checkout develop +cd freqtrade +``` +**2. Create the config file** +Switch `"dry_run": true,` +```bash +cp config.json.example config.json +vi config.json +``` +**3. Build your docker image and run it** +```bash +docker build -t freqtrade . +docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade +``` + ------ ## Automatic Installation - Docker From 5623ea3ac65ff832f5b680abf00592c7e0ba77fd Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 9 Jun 2018 21:44:20 +0200 Subject: [PATCH 011/239] Add forcesell at end of backtest period --- freqtrade/optimize/backtesting.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 028a4f521..2b1bb98a0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -130,6 +130,21 @@ class Backtesting(object): (sell_row.date - buy_row.date).seconds // 60 ), \ sell_row.date + if partial_ticker: + # no sell condition found - trade stil open at end of backtest period + sell_row = partial_ticker[-1] + logger.info('Force_selling still open trade %s with %s perc - %s', pair, + trade.calc_profit_percent(rate=sell_row.close), + trade.calc_profit(rate=sell_row.close)) + return \ + sell_row, \ + ( + pair, + trade.calc_profit_percent(rate=sell_row.close), + trade.calc_profit(rate=sell_row.close), + (sell_row.date - buy_row.date).seconds // 60 + ), \ + sell_row.date return None def backtest(self, args: Dict) -> DataFrame: @@ -170,6 +185,7 @@ class Backtesting(object): ticker_data.drop(ticker_data.head(1).index, inplace=True) + # TODO: why convert from Pandas to list?? ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None @@ -202,6 +218,11 @@ class Backtesting(object): row.date.strftime('%s'), row2.date.strftime('%s'), index, trade_entry[3])) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date + # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: From 24a875ed465442100befb6f07d626e7caeb37a25 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 9 Jun 2018 21:44:57 +0200 Subject: [PATCH 012/239] remove experimental parameters - they are read by analyze.py anyway --- freqtrade/optimize/backtesting.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2b1bb98a0..56a9ca9bc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -277,16 +277,12 @@ class Backtesting(object): ) # Execute backtest and print results - sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False) - use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False) results = self.backtest( { 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, 'realistic': self.config.get('realistic_simulation', False), - 'sell_profit_only': sell_profit_only, - 'use_sell_signal': use_sell_signal, 'record': self.config.get('export'), 'recordfn': self.config.get('exportfilename'), } From 3094acc7fb33559f7212e313677710131af9ec9f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 08:58:28 +0200 Subject: [PATCH 013/239] update comment --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 56a9ca9bc..d6748bd76 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -185,7 +185,8 @@ class Backtesting(object): ticker_data.drop(ticker_data.head(1).index, inplace=True) - # TODO: why convert from Pandas to list?? + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None From c1b2e06edad52dc91602088c67c4a97b05780b16 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 09:07:04 +0200 Subject: [PATCH 014/239] simplify return from _get_sell_trade_entry --- freqtrade/optimize/backtesting.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d6748bd76..acb419ef5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -128,8 +128,7 @@ class Backtesting(object): trade.calc_profit_percent(rate=sell_row.close), trade.calc_profit(rate=sell_row.close), (sell_row.date - buy_row.date).seconds // 60 - ), \ - sell_row.date + ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period sell_row = partial_ticker[-1] @@ -143,8 +142,7 @@ class Backtesting(object): trade.calc_profit_percent(rate=sell_row.close), trade.calc_profit(rate=sell_row.close), (sell_row.date - buy_row.date).seconds // 60 - ), \ - sell_row.date + ) return None def backtest(self, args: Dict) -> DataFrame: @@ -208,8 +206,8 @@ class Backtesting(object): trade_count_lock, args) if ret: - row2, trade_entry, next_date = ret - lock_pair_until = next_date + row2, trade_entry = ret + lock_pair_until = row2.date trades.append(trade_entry) if record: # Note, need to be json.dump friendly From 9c57d3aa8b98c95fbd31fb80e2323fa80bb537ac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:15:25 +0200 Subject: [PATCH 015/239] add BacktestresultTuple --- freqtrade/optimize/backtesting.py | 88 +++++++++++++++++-------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index acb419ef5..574b7b283 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,7 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace -from typing import Dict, Tuple, Any, List, Optional +from typing import Dict, Tuple, Any, List, Optional, NamedTuple import arrow from pandas import DataFrame @@ -23,6 +23,18 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) +class BacktestResult(NamedTuple): + """ + NamedTuple Defining BacktestResults inputs. + """ + pair: str + profit_percent: float + profit_abs: float + open_time: float + close_time: float + trade_duration: float + + class Backtesting(object): """ Backtesting class, this class contains all the logic to run a backtest @@ -73,15 +85,15 @@ class Backtesting(object): headers = ['pair', 'buy count', 'avg profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: - result = results[results.currency == pair] + result = results[results.pair == pair] tabular_data.append([ pair, len(result.index), result.profit_percent.mean() * 100.0, - result.profit_BTC.sum(), - result.duration.mean(), - len(result[result.profit_BTC > 0]), - len(result[result.profit_BTC < 0]) + result.profit_abs.sum(), + result.trade_duration.mean(), + len(result[result.profit_abs > 0]), + len(result[result.profit_abs < 0]) ]) # Append Total @@ -89,16 +101,16 @@ class Backtesting(object): 'TOTAL', len(results.index), results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.duration.mean(), - len(results[results.profit_BTC > 0]), - len(results[results.profit_BTC < 0]) + results.profit_abs.sum(), + results.trade_duration.mean(), + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, - partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[Tuple]: + partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]: stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) @@ -121,28 +133,27 @@ class Backtesting(object): buy_signal = sell_row.buy if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal, sell_row.sell): - return \ - sell_row, \ - ( - pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close), - (sell_row.date - buy_row.date).seconds // 60 - ) + + return BacktestResult(pair=pair, + profit_percent=trade.calc_profit_percent(rate=sell_row.close), + profit_abs=trade.calc_profit(rate=sell_row.close), + open_time=buy_row.date, + close_time=sell_row.date, + trade_duration=(sell_row.date - buy_row.date).seconds // 60 + ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period sell_row = partial_ticker[-1] - logger.info('Force_selling still open trade %s with %s perc - %s', pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close)) - return \ - sell_row, \ - ( - pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close), - (sell_row.date - buy_row.date).seconds // 60 - ) + btr = BacktestResult(pair=pair, + profit_percent=trade.calc_profit_percent(rate=sell_row.close), + profit_abs=trade.calc_profit(rate=sell_row.close), + open_time=buy_row.date, + close_time=sell_row.date, + trade_duration=(sell_row.date - buy_row.date).seconds // 60 + ) + logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, + btr.profit_percent, btr.profit_abs) + return btr return None def backtest(self, args: Dict) -> DataFrame: @@ -202,20 +213,19 @@ class Backtesting(object): trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - ret = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - if ret: - row2, trade_entry = ret - lock_pair_until = row2.date + if trade_entry: + lock_pair_until = trade_entry.close_time trades.append(trade_entry) if record: # Note, need to be json.dump friendly # record a tuple of pair, current_profit_percent, # entry-date, duration - records.append((pair, trade_entry[1], - row.date.strftime('%s'), - row2.date.strftime('%s'), + records.append((pair, trade_entry.profit_percent, + trade_entry.open_time.strftime('%s'), + trade_entry.close_time.strftime('%s'), index, trade_entry[3])) else: # Set lock_pair_until to end of testing period if trade could not be closed @@ -228,7 +238,7 @@ class Backtesting(object): logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] - return DataFrame.from_records(trades, columns=labels) + return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: """ From 7b5a2946e5b8636a4fd6d181b976951b03ede96f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:19:32 +0200 Subject: [PATCH 016/239] adjust for forcesell backtesting --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 33d9703de..72de5eb9c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -538,7 +538,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): backtesting.populate_buy_trend = _trend_alternate # Override backtesting.populate_sell_trend = _trend_alternate # Override results = backtesting.backtest(backtest_conf) - assert len(results) == 3 + assert len(results) == 4 def test_backtest_record(default_conf, fee, mocker): From c9476fade8180bb57b3fb7f123a25badd174a48d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:20:41 +0200 Subject: [PATCH 017/239] adjust tests for forcesell --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 72de5eb9c..ad328edc2 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -478,7 +478,7 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) - tests = [['raise', 17], ['lower', 0], ['sine', 16]] + tests = [['raise', 18], ['lower', 0], ['sine', 16]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres, mocker) From 17c0ceec04b8cb9f826a80bee1a321bef77874b1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:22:24 +0200 Subject: [PATCH 018/239] adjust tests for backtestresult type --- freqtrade/tests/optimize/test_backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index ad328edc2..d5ac6e01e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -353,10 +353,10 @@ def test_generate_text_table(default_conf, mocker): results = pd.DataFrame( { - 'currency': ['ETH/BTC', 'ETH/BTC'], + 'pair': ['ETH/BTC', 'ETH/BTC'], 'profit_percent': [0.1, 0.2], - 'profit_BTC': [0.2, 0.4], - 'duration': [10, 30], + 'profit_abs': [0.2, 0.4], + 'trade_duration': [10, 30], 'profit': [2, 0], 'loss': [0, 0] } From 322a528c12aacc4712c5d0685d1d582dd3e6747a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:25:16 +0200 Subject: [PATCH 019/239] fix bug with backtestResult --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 574b7b283..cd020a6db 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -226,7 +226,7 @@ class Backtesting(object): records.append((pair, trade_entry.profit_percent, trade_entry.open_time.strftime('%s'), trade_entry.close_time.strftime('%s'), - index, trade_entry[3])) + index, trade_entry.trade_duration)) else: # Set lock_pair_until to end of testing period if trade could not be closed # This happens only if the buy-signal was with the last candle From aff1ede46bf88817e402d090a289bb39b699e291 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:25:52 +0200 Subject: [PATCH 020/239] Fix last backtesting test --- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index d5ac6e01e..7c617c38f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -560,12 +560,12 @@ def test_backtest_record(default_conf, fee, mocker): backtesting.populate_buy_trend = _trend_alternate # Override backtesting.populate_sell_trend = _trend_alternate # Override results = backtesting.backtest(backtest_conf) - assert len(results) == 3 + assert len(results) == 4 # Assert file_dump_json was only called once assert names == ['backtest-result.json'] records = records[0] # Ensure records are of correct type - assert len(records) == 3 + assert len(records) == 4 # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None From 31025216f94f58f217841f1215a7869a684bbd2c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:32:07 +0200 Subject: [PATCH 021/239] fix type of open/close timestmap --- freqtrade/optimize/backtesting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cd020a6db..68480c997 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,6 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace +from datetime import datetime from typing import Dict, Tuple, Any, List, Optional, NamedTuple import arrow @@ -30,8 +31,8 @@ class BacktestResult(NamedTuple): pair: str profit_percent: float profit_abs: float - open_time: float - close_time: float + open_time: datetime + close_time: datetime trade_duration: float From b81588307f0e14d6e6ec82073e86f4e7f98cbaf7 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:37:53 +0200 Subject: [PATCH 022/239] Add "open_at_end" parameter --- freqtrade/optimize/backtesting.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 68480c997..f87340829 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -34,6 +34,7 @@ class BacktestResult(NamedTuple): open_time: datetime close_time: datetime trade_duration: float + open_at_end: bool class Backtesting(object): @@ -140,7 +141,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.close), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60 + trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_at_end=False ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -150,7 +152,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.close), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60 + trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_at_end=True ) logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) From 1cd7ac55a8d742f7c7e2278685cdc412205eacd0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:45:16 +0200 Subject: [PATCH 023/239] Added "left open trades" report --- freqtrade/optimize/backtesting.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f87340829..cc20e9789 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -311,6 +311,17 @@ class Backtesting(object): ) ) + logger.info( + '\n==================================== ' + 'LEFT OPEN TRADES REPORT' + ' ====================================\n' + '%s', + self._generate_text_table( + data, + results.loc[results.open_at_end == True] + ) + ) + def setup_configuration(args: Namespace) -> Dict[str, Any]: """ From 27ee8f73604b52da8e7a5865d52949afd9b99a7b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:55:48 +0200 Subject: [PATCH 024/239] make flake happy --- freqtrade/optimize/backtesting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cc20e9789..e526a6ec4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -241,7 +241,6 @@ class Backtesting(object): if record and record.find('trades') >= 0: logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: @@ -318,7 +317,7 @@ class Backtesting(object): '%s', self._generate_text_table( data, - results.loc[results.open_at_end == True] + results.loc[results.open_at_end] ) ) From 4710210cffe52fcac5406d1d1840732253c1073f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:56:10 +0200 Subject: [PATCH 025/239] fix hyperopt to use new backtesting result tuple --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 878acc2dc..e952458a7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -451,7 +451,7 @@ class Hyperopt(Backtesting): total_profit = results.profit_percent.sum() trade_count = len(results.index) - trade_duration = results.duration.mean() + trade_duration = results.trade_duration.mean() if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') @@ -488,10 +488,10 @@ class Hyperopt(Backtesting): 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), + results.profit_abs.sum(), self.config['stake_currency'], results.profit_percent.sum(), - results.duration.mean(), + results.trade_duration.mean(), ) def start(self) -> None: From 9cc087c788bd56fc230d8bafa6993886d7ceb514 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:56:23 +0200 Subject: [PATCH 026/239] update hyperopt tests to support new structure --- freqtrade/tests/optimize/test_hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 3edfe4393..62fd17b82 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -400,7 +400,7 @@ def test_format_results(init_hyperopt): ('LTC/BTC', 1, 1, 123), ('XPR/BTC', -1, -2, -246) ] - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration'] df = pd.DataFrame.from_records(trades, columns=labels) result = _HYPEROPT.format_results(df) @@ -530,7 +530,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: trades = [ ('POWR/BTC', 0.023117, 0.000233, 100) ] - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration'] backtest_result = pd.DataFrame.from_records(trades, columns=labels) mocker.patch( From 12e455cbf5616462d0ca198eb2122e13ba17add8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 20:52:42 +0200 Subject: [PATCH 027/239] add buy/sell index to backtest result --- freqtrade/optimize/backtesting.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e526a6ec4..ffd6e2768 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,6 +33,8 @@ class BacktestResult(NamedTuple): profit_abs: float open_time: datetime close_time: datetime + open_index: int + close_index: int trade_duration: float open_at_end: bool @@ -142,6 +144,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_index=buy_row.index, + close_index=sell_row.index, open_at_end=False ) if partial_ticker: @@ -153,6 +157,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_index=buy_row.index, + close_index=sell_row.index, open_at_end=True ) logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, From 367601518481092ae92d320e0eeabaa4b2bf7b8a Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 11 Jun 2018 16:21:57 +0300 Subject: [PATCH 028/239] Fix check --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 70c74755e..8c8017d0b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,7 +7,7 @@ import logging import time import traceback from datetime import datetime -from typing import Dict, List, Optional, Any, Callable +from typing import Dict, List, Optional, Any, Callable, Optional import arrow import requests @@ -239,7 +239,7 @@ class FreqtradeBot(object): return final_list - def get_target_bid(self, ticker: Dict[str, float]) -> float: + def get_target_bid(self, ticker: Dict[str, float]) -> Optional[float]: """ Calculates bid target between current ask price and last price :param ticker: Ticker to use for getting Ask and Last Price From 90025d0ac437fafe261df6d588f0732f3698bbb4 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 11 Jun 2018 16:38:10 +0300 Subject: [PATCH 029/239] Fix check --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ab22b2826..1e32aecdf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,7 +7,7 @@ import logging import time import traceback from datetime import datetime -from typing import Dict, List, Optional, Any, Callable, Optional +from typing import Dict, List, Optional, Any, Callable import arrow import requests From 708320318c5a340d70a161177725f670764d0bb9 Mon Sep 17 00:00:00 2001 From: Anton Date: Tue, 12 Jun 2018 01:05:43 +0300 Subject: [PATCH 030/239] Check minimal amount --- freqtrade/freqtradebot.py | 12 +++++++++--- freqtrade/tests/test_freqtradebot.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1e32aecdf..1477a931e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -234,7 +234,7 @@ class FreqtradeBot(object): return final_list - def get_target_bid(self, ticker: Dict[str, float]) -> Optional[float]: + def get_target_bid(self, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price :param ticker: Ticker to use for getting Ask and Last Price @@ -245,7 +245,7 @@ class FreqtradeBot(object): balance = self.config['bid_strategy']['ask_last_balance'] return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) - def _get_trade_stake_amount(self) -> float: + def _get_trade_stake_amount(self) -> Optional[float]: stake_amount = self.config['stake_amount'] avaliable_amount = exchange.get_balance(self.config['stake_currency']) @@ -254,7 +254,13 @@ class FreqtradeBot(object): if open_trades >= self.config['max_open_trades']: logger.warning('Can\'t open a new trade: max number of trades is reached') return None - return avaliable_amount / (self.config['max_open_trades'] - open_trades) + trade_stake_amount = avaliable_amount / (self.config['max_open_trades'] - open_trades) + if trade_stake_amount < 0.0005: + raise DependencyException( + 'Available balance(%f %s) is lower than minimal' % ( + avaliable_amount, self.config['stake_currency']) + ) + return trade_stake_amount # Check if stake_amount is fulfilled if avaliable_amount < stake_amount: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 314d880b6..80b66735d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -249,11 +249,21 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) ) + # test defined stake amount freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade._get_trade_stake_amount() + # test UNLIMITED_STAKE_AMOUNT + conf = deepcopy(default_conf) + conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + conf['max_open_trades'] = 2 + freqtrade = FreqtradeBot(conf) + + with pytest.raises(DependencyException, match=r'.*is lower than minimal.*'): + freqtrade._get_trade_stake_amount() + def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, From bfde33c945f4374fc8ca69075d41a2d9c731d6e9 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 21:12:55 +0200 Subject: [PATCH 031/239] Use timestamp() instead of strftime this will avoid a bug shifting epoch time by 1 hour: https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffd6e2768..f7fade102 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -234,8 +234,8 @@ class Backtesting(object): # record a tuple of pair, current_profit_percent, # entry-date, duration records.append((pair, trade_entry.profit_percent, - trade_entry.open_time.strftime('%s'), - trade_entry.close_time.strftime('%s'), + trade_entry.open_time.timestamp(), + trade_entry.close_time.timestamp(), index, trade_entry.trade_duration)) else: # Set lock_pair_until to end of testing period if trade could not be closed From 1f6b9c332b5d3f38bed26dd539ef29d44fae6ae1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 21:38:14 +0200 Subject: [PATCH 032/239] fix default datadir not working in plot-script --- scripts/plot_profit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 803bf71de..012446065 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -121,7 +121,7 @@ def plot_profit(args: Namespace) -> None: logger.info('Filter, keep pairs %s' % pairs) tickers = optimize.load_data( - datadir=args.datadir, + datadir=config.get('datadir'), pairs=pairs, ticker_interval=tick_interval, refresh_pairs=False, From 182f4c603be87097cfd9b8229f6db688171250a7 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 21:43:14 +0200 Subject: [PATCH 033/239] fix plot-script datadir not working --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 122c002a8..ce1a4b819 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -91,7 +91,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data( - datadir=args.datadir, + datadir=_CONF.get("datadir"), pairs=[pair], ticker_interval=tick_interval, refresh_pairs=_CONF.get('refresh_pairs', False), From e3ced7c15e0d1dc0582485147d59ca5122efebfc Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 22:29:30 +0200 Subject: [PATCH 034/239] extract export from backtest function --- freqtrade/optimize/backtesting.py | 47 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f7fade102..1146d6b00 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -112,6 +112,21 @@ class Backtesting(object): ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: + + records = [] + print(results) + for index, trade_entry in results.iterrows(): + pass + records.append((trade_entry.pair, trade_entry.profit_percent, + trade_entry.open_time.timestamp(), + trade_entry.close_time.timestamp(), + trade_entry.open_index - 1, trade_entry.trade_duration)) + + if records: + logger.info('Dumping backtest results to %s', recordfilename) + file_dump_json(recordfilename, records) + def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]: @@ -144,8 +159,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, - open_index=buy_row.index, - close_index=sell_row.index, + open_index=buy_row.Index, + close_index=sell_row.Index, open_at_end=False ) if partial_ticker: @@ -157,8 +172,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, - open_index=buy_row.index, - close_index=sell_row.index, + open_index=buy_row.Index, + close_index=sell_row.Index, open_at_end=True ) logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, @@ -179,17 +194,12 @@ class Backtesting(object): processed: a processed dictionary with format {pair, data} max_open_trades: maximum number of concurrent trades (default: 0, disabled) realistic: do we try to simulate realistic trades? (default: True) - sell_profit_only: sell if profit only - use_sell_signal: act on sell-signal :return: DataFrame """ headers = ['date', 'buy', 'open', 'close', 'sell'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) - record = args.get('record', None) - recordfilename = args.get('recordfn', 'backtest-result.json') - records = [] trades = [] trade_count_lock: Dict = {} for pair, pair_data in processed.items(): @@ -229,24 +239,11 @@ class Backtesting(object): if trade_entry: lock_pair_until = trade_entry.close_time trades.append(trade_entry) - if record: - # Note, need to be json.dump friendly - # record a tuple of pair, current_profit_percent, - # entry-date, duration - records.append((pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - index, trade_entry.trade_duration)) else: # Set lock_pair_until to end of testing period if trade could not be closed # This happens only if the buy-signal was with the last candle lock_pair_until = ticker_data.iloc[-1].date - # For now export inside backtest(), maybe change so that backtest() - # returns a tuple like: (dataframe, records, logs, etc) - if record and record.find('trades') >= 0: - logger.info('Dumping backtest results to %s', recordfilename) - file_dump_json(recordfilename, records) return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: @@ -301,10 +298,12 @@ class Backtesting(object): 'processed': preprocessed, 'max_open_trades': max_open_trades, 'realistic': self.config.get('realistic_simulation', False), - 'record': self.config.get('export'), - 'recordfn': self.config.get('exportfilename'), } ) + + if self.config.get('export', False): + self._store_backtest_result(self.config.get('exportfilename'), results) + logger.info( '\n==================================== ' 'BACKTESTING REPORT' From 8d8e6dcffc60cabd6c135d95337401d9abb5b69b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:31:42 +0200 Subject: [PATCH 035/239] Add test for extracted backtest_results test --- freqtrade/tests/optimize/test_backtesting.py | 26 +++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 7c617c38f..e0342851f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -550,16 +550,24 @@ def test_backtest_record(default_conf, fee, mocker): 'freqtrade.optimize.backtesting.file_dump_json', new=lambda n, r: (names.append(n), records.append(r)) ) - backtest_conf = _make_backtest_conf( - mocker, - conf=default_conf, - pair='UNITTEST/BTC', - record="trades" - ) + backtesting = Backtesting(default_conf) - backtesting.populate_buy_trend = _trend_alternate # Override - backtesting.populate_sell_trend = _trend_alternate # Override - results = backtesting.backtest(backtest_conf) + results = pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_index": [1, 119, 153, 185], + "close_index": [118, 151, 184, 199], + "trade_duration": [123, 34, 31, 14]}) + backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 # Assert file_dump_json was only called once assert names == ['backtest-result.json'] From 0f117d480e4f8822c76bbda2e4cd24300b495f50 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:42:24 +0200 Subject: [PATCH 036/239] improve backtesting-tests * assert length of result specifically * add assert for "open_at_end" --- freqtrade/tests/optimize/test_backtesting.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e0342851f..15f1d978e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -435,6 +435,7 @@ def test_backtest(default_conf, fee, mocker) -> None: } ) assert not results.empty + assert len(results) == 2 def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: @@ -457,6 +458,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: } ) assert not results.empty + assert len(results) == 1 def test_processed(default_conf, mocker) -> None: @@ -538,7 +540,10 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): backtesting.populate_buy_trend = _trend_alternate # Override backtesting.populate_sell_trend = _trend_alternate # Override results = backtesting.backtest(backtest_conf) + backtesting._store_backtest_result("test_.json", results) assert len(results) == 4 + # One trade was force-closed at the end + assert len(results.loc[results.open_at_end]) == 1 def test_backtest_record(default_conf, fee, mocker): From 6e68c3b230490e47cb84b27ba1ebb9fd176d32b9 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:52:17 +0200 Subject: [PATCH 037/239] fix backtesting.md formatting --- docs/backtesting.md | 52 +++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 8364d77e4..e46999d4b 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -1,17 +1,19 @@ # Backtesting + This page explains how to validate your strategy performance by using Backtesting. ## Table of Contents + - [Test your strategy with Backtesting](#test-your-strategy-with-backtesting) - [Understand the backtesting result](#understand-the-backtesting-result) ## Test your strategy with Backtesting + Now you have good Buy and Sell strategies, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). - Backtesting will use the crypto-currencies (pair) from your config file and load static tickers located in [/freqtrade/tests/testdata](https://github.com/freqtrade/freqtrade/tree/develop/freqtrade/tests/testdata). @@ -19,70 +21,80 @@ If the 5 min and 1 min ticker for the crypto-currencies to test is not already in the `testdata` folder, backtesting will download them automatically. Testdata files will not be updated until you specify it. -The result of backtesting will confirm you if your bot as more chance to -make a profit than a loss. - +The result of backtesting will confirm you if your bot as more chance to make a profit than a loss. The backtesting is very easy with freqtrade. ### Run a backtesting against the currencies listed in your config file -**With 5 min tickers (Per default)** +#### With 5 min tickers (Per default) + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation ``` -**With 1 min tickers** +#### With 1 min tickers + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --ticker-interval 1m ``` -**Update cached pairs with the latest data** +#### Update cached pairs with the latest data + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-cached ``` -**With live data (do not alter your testdata files)** +#### With live data (do not alter your testdata files) + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --live ``` -**Using a different on-disk ticker-data source** +#### Using a different on-disk ticker-data source + ```bash python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101 ``` -**With a (custom) strategy file** +#### With a (custom) strategy file + ```bash python3 ./freqtrade/main.py -s TestStrategy backtesting ``` + Where `-s TestStrategy` refers to the class name within the strategy file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory -**Exporting trades to file** +#### Exporting trades to file + ```bash python3 ./freqtrade/main.py backtesting --export trades ``` -**Exporting trades to file specifying a custom filename** +#### Exporting trades to file specifying a custom filename + ```bash python3 ./freqtrade/main.py backtesting --export trades --export-filename=backtest_teststrategy.json ``` +#### Running backtest with smaller testset -**Running backtest with smaller testset** Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: + ```bash python3 ./freqtrade/main.py backtesting --timerange=-200 ``` -***Advanced use of timerange*** +#### Advanced use of timerange + Doing `--timerange=-200` will get the last 200 timeframes from your inputdata. You can also specify specific dates, or a range span indexed by start and stop. The full timerange specification: + - Use last 123 tickframes of data: `--timerange=-123` - Use first 123 tickframes of data: `--timerange=123-` - Use tickframes from line 123 through 456: `--timerange=123-456` @@ -92,11 +104,12 @@ The full timerange specification: - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` +#### Downloading new set of ticker data -**Downloading new set of ticker data** To download new set of backtesting ticker data, you can use a download script. If you are using Binance for example: + - create a folder `user_data/data/binance` and copy `pairs.json` in that folder. - update the `pairs.json` to contain the currency pairs you are interested in. @@ -119,14 +132,14 @@ This will download ticker data for all the currency pairs you defined in `pairs. - To download ticker data for only 10 days, use `--days 10`. - Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. - -For help about backtesting usage, please refer to -[Backtesting commands](#backtesting-commands). +For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). ## Understand the backtesting result + The most important in the backtesting is to understand the result. A backtesting result will look like that: + ``` ====================== BACKTESTING REPORT ================================ pair buy count avg profit % total profit BTC avg duration @@ -146,6 +159,7 @@ TOTAL 419 -0.41 -0.00348593 52.9 The last line will give you the overall performance of your strategy, here: + ``` TOTAL 419 -0.41 -0.00348593 52.9 ``` @@ -161,6 +175,7 @@ strategy, your sell strategy, and also by the `minimal_roi` and As for an example if your minimal_roi is only `"0": 0.01`. You cannot expect the bot to make more profit than 1% (because it will sell every time a trade will reach 1%). + ```json "minimal_roi": { "0": 0.01 @@ -173,6 +188,7 @@ profit. Hence, keep in mind that your performance is a mix of your strategies, your configuration, and the crypto-currency you have set up. ## Next step + Great, your strategy is profitable. What if the bot can give your the optimal parameters to use for your strategy? Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) From 6357812743defe2cd3208d00dc37b4415409a841 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:57:49 +0200 Subject: [PATCH 038/239] fix backtest report able --- freqtrade/optimize/backtesting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1146d6b00..5df4fb28a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -305,9 +305,9 @@ class Backtesting(object): self._store_backtest_result(self.config.get('exportfilename'), results) logger.info( - '\n==================================== ' + '\n======================================== ' 'BACKTESTING REPORT' - ' ====================================\n' + ' =========================================\n' '%s', self._generate_text_table( data, @@ -316,9 +316,9 @@ class Backtesting(object): ) logger.info( - '\n==================================== ' + '\n====================================== ' 'LEFT OPEN TRADES REPORT' - ' ====================================\n' + ' ======================================\n' '%s', self._generate_text_table( data, From e22da45474aee9487d2e2a38556bcac987bd88b2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 07:00:05 +0200 Subject: [PATCH 039/239] update documentation with forcesell at the end of the backtest period --- docs/backtesting.md | 49 ++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index e46999d4b..127b4ee20 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -141,22 +141,43 @@ The most important in the backtesting is to understand the result. A backtesting result will look like that: ``` -====================== BACKTESTING REPORT ================================ -pair buy count avg profit % total profit BTC avg duration --------- ----------- -------------- ------------------ -------------- -ETH/BTC 56 -0.67 -0.00075455 62.3 -LTC/BTC 38 -0.48 -0.00036315 57.9 -ETC/BTC 42 -1.15 -0.00096469 67.0 -DASH/BTC 72 -0.62 -0.00089368 39.9 -ZEC/BTC 45 -0.46 -0.00041387 63.2 -XLM/BTC 24 -0.88 -0.00041846 47.7 -NXT/BTC 24 0.68 0.00031833 40.2 -POWR/BTC 35 0.98 0.00064887 45.3 -ADA/BTC 43 -0.39 -0.00032292 55.0 -XMR/BTC 40 -0.40 -0.00032181 47.4 -TOTAL 419 -0.41 -0.00348593 52.9 +======================================== BACKTESTING REPORT ========================================= +| pair | buy count | avg profit % | total profit BTC | avg duration | profit | loss | +|:---------|------------:|---------------:|-------------------:|---------------:|---------:|-------:| +| ETH/BTC | 44 | 0.18 | 0.00159118 | 50.9 | 44 | 0 | +| LTC/BTC | 27 | 0.10 | 0.00051931 | 103.1 | 26 | 1 | +| ETC/BTC | 24 | 0.05 | 0.00022434 | 166.0 | 22 | 2 | +| DASH/BTC | 29 | 0.18 | 0.00103223 | 192.2 | 29 | 0 | +| ZEC/BTC | 65 | -0.02 | -0.00020621 | 202.7 | 62 | 3 | +| XLM/BTC | 35 | 0.02 | 0.00012877 | 242.4 | 32 | 3 | +| BCH/BTC | 12 | 0.62 | 0.00149284 | 50.0 | 12 | 0 | +| POWR/BTC | 21 | 0.26 | 0.00108215 | 134.8 | 21 | 0 | +| ADA/BTC | 54 | -0.19 | -0.00205202 | 191.3 | 47 | 7 | +| XMR/BTC | 24 | -0.43 | -0.00206013 | 120.6 | 20 | 4 | +| TOTAL | 335 | 0.03 | 0.00175246 | 157.9 | 315 | 20 | +2018-06-13 06:57:27,347 - freqtrade.optimize.backtesting - INFO - +====================================== LEFT OPEN TRADES REPORT ====================================== +| pair | buy count | avg profit % | total profit BTC | avg duration | profit | loss | +|:---------|------------:|---------------:|-------------------:|---------------:|---------:|-------:| +| ETH/BTC | 3 | 0.16 | 0.00009619 | 25.0 | 3 | 0 | +| LTC/BTC | 1 | -1.00 | -0.00020118 | 1085.0 | 0 | 1 | +| ETC/BTC | 2 | -1.80 | -0.00071933 | 1092.5 | 0 | 2 | +| DASH/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| ZEC/BTC | 3 | -4.27 | -0.00256826 | 1301.7 | 0 | 3 | +| XLM/BTC | 3 | -1.11 | -0.00066744 | 965.0 | 0 | 3 | +| BCH/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| POWR/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| ADA/BTC | 7 | -3.58 | -0.00503604 | 850.0 | 0 | 7 | +| XMR/BTC | 4 | -3.79 | -0.00303456 | 291.2 | 0 | 4 | +| TOTAL | 23 | -2.63 | -0.01213062 | 750.4 | 3 | 20 | + ``` +The 1st table will contain all trades the bot made. + +The 2nd table will contain all trades the bot had to `forcesell` at the end of the backtest period to prsent a full picture. +These trades are also included in the first table, but are extracted separately for clarity. + The last line will give you the overall performance of your strategy, here: From cddb062db52252a50b519890f882def8e0aada54 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Jun 2018 03:49:09 +0200 Subject: [PATCH 040/239] save rpc instances only in registered_modules, add some abstract methods --- freqtrade/rpc/rpc.py | 17 +++++++- freqtrade/rpc/rpc_manager.py | 48 ++++++++-------------- freqtrade/rpc/telegram.py | 52 ++++++++++++++---------- freqtrade/tests/rpc/test_rpc_manager.py | 38 +++++------------ freqtrade/tests/rpc/test_rpc_telegram.py | 42 +++++++++---------- freqtrade/tests/test_freqtradebot.py | 2 +- 6 files changed, 95 insertions(+), 104 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 33cfc3e8f..5ac78f2cb 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -2,21 +2,21 @@ This module contains class to define a RPC communications """ import logging +from abc import abstractmethod from datetime import datetime, timedelta, date from decimal import Decimal from typing import Dict, Tuple, Any import arrow import sqlalchemy as sql -from pandas import DataFrame from numpy import mean, nan_to_num +from pandas import DataFrame from freqtrade import exchange from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State - logger = logging.getLogger(__name__) @@ -32,6 +32,19 @@ class RPC(object): """ self.freqtrade = freqtrade + @abstractmethod + def cleanup(self) -> str: + """ Cleanup pending module resources """ + + @property + @abstractmethod + def name(self) -> None: + """ Returns the lowercase name of this module """ + + @abstractmethod + def send_msg(self, msg: str) -> None: + """ Sends a message to all registered rpc modules """ + def rpc_trade_status(self) -> Tuple[bool, Any]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 58e9bf2b9..ce01b78a3 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,12 +1,12 @@ """ This module contains class to manage RPC communications (Telegram, Slack, ...) """ -from typing import Any, List import logging +from typing import List +from freqtrade.rpc.rpc import RPC from freqtrade.rpc.telegram import Telegram - logger = logging.getLogger(__name__) @@ -15,36 +15,21 @@ class RPCManager(object): Class to manage RPC objects (Telegram, Slack, ...) """ def __init__(self, freqtrade) -> None: - """ - Initializes all enabled rpc modules - :param config: config to use - :return: None - """ - self.freqtrade = freqtrade + """ Initializes all enabled rpc modules """ + self.registered_modules: List[RPC] = [] - self.registered_modules: List[str] = [] - self.telegram: Any = None - self._init() - - def _init(self) -> None: - """ - Init RPC modules - :return: - """ - if self.freqtrade.config['telegram'].get('enabled', False): + # Enable telegram + if freqtrade.config['telegram'].get('enabled', False): logger.info('Enabling rpc.telegram ...') - self.registered_modules.append('telegram') - self.telegram = Telegram(self.freqtrade) + self.registered_modules.append(Telegram(freqtrade)) def cleanup(self) -> None: - """ - Stops all enabled rpc modules - :return: None - """ - if 'telegram' in self.registered_modules: - logger.info('Cleaning up rpc.telegram ...') - self.registered_modules.remove('telegram') - self.telegram.cleanup() + """ Stops all enabled rpc modules """ + for mod in self.registered_modules: + logger.info('Cleaning up rpc.%s ...', mod.name) + mod.cleanup() + + self.registered_modules = [] def send_msg(self, msg: str) -> None: """ @@ -52,6 +37,7 @@ class RPCManager(object): :param msg: message :return: None """ - logger.info(msg) - if 'telegram' in self.registered_modules: - self.telegram.send_msg(msg) + logger.info('Sending rpc message: %s', msg) + for mod in self.registered_modules: + logger.debug('Forwarding message to rpc.%s', mod.name) + mod.send_msg(msg) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 43383fe43..c00ba6a8e 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -14,7 +14,6 @@ from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ from freqtrade.rpc.rpc import RPC - logger = logging.getLogger(__name__) @@ -57,6 +56,11 @@ class Telegram(RPC): """ Telegram, this class send messages to Telegram """ + + @property + def name(self) -> str: + return "telegram" + def __init__(self, freqtrade) -> None: """ Init the Telegram call, and init the super class RPC @@ -120,6 +124,10 @@ class Telegram(RPC): self._updater.stop() + def send_msg(self, msg: str) -> None: + """ Send a message to telegram channel """ + self._send_msg(msg) + def is_enabled(self) -> bool: """ Returns True if the telegram module is activated, False otherwise @@ -146,10 +154,10 @@ class Telegram(RPC): # Fetch open trade (error, trades) = self.rpc_trade_status() if error: - self.send_msg(trades, bot=bot) + self._send_msg(trades, bot=bot) else: for trademsg in trades: - self.send_msg(trademsg, bot=bot) + self._send_msg(trademsg, bot=bot) @authorized_only def _status_table(self, bot: Bot, update: Update) -> None: @@ -163,12 +171,12 @@ class Telegram(RPC): # Fetch open trade (err, df_statuses) = self.rpc_status_table() if err: - self.send_msg(df_statuses, bot=bot) + self._send_msg(df_statuses, bot=bot) else: message = tabulate(df_statuses, headers='keys', tablefmt='simple') message = "
{}
".format(message) - self.send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) @authorized_only def _daily(self, bot: Bot, update: Update) -> None: @@ -189,7 +197,7 @@ class Telegram(RPC): self._config['fiat_display_currency'] ) if error: - self.send_msg(stats, bot=bot) + self._send_msg(stats, bot=bot) else: stats = tabulate(stats, headers=[ @@ -203,7 +211,7 @@ class Telegram(RPC): timescale, stats ) - self.send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) @authorized_only def _profit(self, bot: Bot, update: Update) -> None: @@ -219,7 +227,7 @@ class Telegram(RPC): self._config['fiat_display_currency'] ) if error: - self.send_msg(stats, bot=bot) + self._send_msg(stats, bot=bot) return # Message to display @@ -250,7 +258,7 @@ class Telegram(RPC): best_pair=stats['best_pair'], best_rate=stats['best_rate'] ) - self.send_msg(markdown_msg, bot=bot) + self._send_msg(markdown_msg, bot=bot) @authorized_only def _balance(self, bot: Bot, update: Update) -> None: @@ -259,7 +267,7 @@ class Telegram(RPC): """ (error, result) = self.rpc_balance(self._config['fiat_display_currency']) if error: - self.send_msg('`All balances are zero.`') + self._send_msg('`All balances are zero.`') return (currencys, total, symbol, value) = result @@ -274,7 +282,7 @@ class Telegram(RPC): output += "\n*Estimated Value*:\n" \ "\t`BTC: {0: .8f}`\n" \ "\t`{1}: {2: .2f}`\n".format(total, symbol, value) - self.send_msg(output) + self._send_msg(output) @authorized_only def _start(self, bot: Bot, update: Update) -> None: @@ -287,7 +295,7 @@ class Telegram(RPC): """ (error, msg) = self.rpc_start() if error: - self.send_msg(msg, bot=bot) + self._send_msg(msg, bot=bot) @authorized_only def _stop(self, bot: Bot, update: Update) -> None: @@ -299,7 +307,7 @@ class Telegram(RPC): :return: None """ (error, msg) = self.rpc_stop() - self.send_msg(msg, bot=bot) + self._send_msg(msg, bot=bot) @authorized_only def _reload_conf(self, bot: Bot, update: Update) -> None: @@ -326,7 +334,7 @@ class Telegram(RPC): trade_id = update.message.text.replace('/forcesell', '').strip() (error, message) = self.rpc_forcesell(trade_id) if error: - self.send_msg(message, bot=bot) + self._send_msg(message, bot=bot) return @authorized_only @@ -340,7 +348,7 @@ class Telegram(RPC): """ (error, trades) = self.rpc_performance() if error: - self.send_msg(trades, bot=bot) + self._send_msg(trades, bot=bot) return stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( @@ -350,7 +358,7 @@ class Telegram(RPC): count=trade['count'] ) for i, trade in enumerate(trades)) message = 'Performance:\n{}'.format(stats) - self.send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) @authorized_only def _count(self, bot: Bot, update: Update) -> None: @@ -363,7 +371,7 @@ class Telegram(RPC): """ (error, trades) = self.rpc_count() if error: - self.send_msg(trades, bot=bot) + self._send_msg(trades, bot=bot) return message = tabulate({ @@ -373,7 +381,7 @@ class Telegram(RPC): }, headers=['current', 'max', 'total stake'], tablefmt='simple') message = "
{}
".format(message) logger.debug(message) - self.send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) @authorized_only def _help(self, bot: Bot, update: Update) -> None: @@ -399,7 +407,7 @@ class Telegram(RPC): "*/help:* `This help message`\n" \ "*/version:* `Show version`" - self.send_msg(message, bot=bot) + self._send_msg(message, bot=bot) @authorized_only def _version(self, bot: Bot, update: Update) -> None: @@ -410,10 +418,10 @@ class Telegram(RPC): :param update: message update :return: None """ - self.send_msg('*Version:* `{}`'.format(__version__), bot=bot) + self._send_msg('*Version:* `{}`'.format(__version__), bot=bot) - def send_msg(self, msg: str, bot: Bot = None, - parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + def _send_msg(self, msg: str, bot: Bot = None, + parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: """ Send given markdown message :param msg: message diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 1d56dea3a..6c073a251 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -7,49 +7,35 @@ from copy import deepcopy from unittest.mock import MagicMock from freqtrade.rpc.rpc_manager import RPCManager -from freqtrade.rpc.telegram import Telegram from freqtrade.tests.conftest import log_has, get_patched_freqtradebot def test_rpc_manager_object() -> None: - """ - Test the Arguments object has the mandatory methods - :return: None - """ - assert hasattr(RPCManager, '_init') + """ Test the Arguments object has the mandatory methods """ assert hasattr(RPCManager, 'send_msg') assert hasattr(RPCManager, 'cleanup') def test__init__(mocker, default_conf) -> None: - """ - Test __init__() method - """ - init_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager._init', MagicMock()) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + """ Test __init__() method """ + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False - rpc_manager = RPCManager(freqtradebot) - assert rpc_manager.freqtrade == freqtradebot + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) assert rpc_manager.registered_modules == [] - assert rpc_manager.telegram is None - assert init_mock.call_count == 1 def test_init_telegram_disabled(mocker, default_conf, caplog) -> None: - """ - Test _init() method with Telegram disabled - """ + """ Test _init() method with Telegram disabled """ caplog.set_level(logging.DEBUG) conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - freqtradebot = get_patched_freqtradebot(mocker, conf) - rpc_manager = RPCManager(freqtradebot) + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) assert not log_has('Enabling rpc.telegram ...', caplog.record_tuples) assert rpc_manager.registered_modules == [] - assert rpc_manager.telegram is None def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: @@ -59,14 +45,12 @@ def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - rpc_manager = RPCManager(freqtradebot) + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert log_has('Enabling rpc.telegram ...', caplog.record_tuples) len_modules = len(rpc_manager.registered_modules) assert len_modules == 1 - assert 'telegram' in rpc_manager.registered_modules - assert isinstance(rpc_manager.telegram, Telegram) + assert 'telegram' in [mod.name for mod in rpc_manager.registered_modules] def test_cleanup_telegram_disabled(mocker, default_conf, caplog) -> None: @@ -99,11 +83,11 @@ def test_cleanup_telegram_enabled(mocker, default_conf, caplog) -> None: rpc_manager = RPCManager(freqtradebot) # Check we have Telegram as a registered modules - assert 'telegram' in rpc_manager.registered_modules + assert 'telegram' in [mod.name for mod in rpc_manager.registered_modules] rpc_manager.cleanup() assert log_has('Cleaning up rpc.telegram ...', caplog.record_tuples) - assert 'telegram' not in rpc_manager.registered_modules + assert 'telegram' not in [mod.name for mod in rpc_manager.registered_modules] assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 0919455ad..47ccf4243 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -258,7 +258,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: _init=MagicMock(), rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), _status_table=status_table, - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -296,7 +296,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _status_table=status_table, - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -341,7 +341,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -397,7 +397,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -465,7 +465,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -506,7 +506,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -604,7 +604,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) @@ -634,7 +634,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) @@ -656,7 +656,7 @@ def test_start_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -680,7 +680,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -705,7 +705,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -730,7 +730,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -898,7 +898,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) @@ -940,7 +940,7 @@ def test_performance_handle(default_conf, update, ticker, fee, mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -981,7 +981,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) @@ -1004,7 +1004,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -1047,7 +1047,7 @@ def test_help_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) @@ -1067,7 +1067,7 @@ def test_version_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) @@ -1090,12 +1090,12 @@ def test_send_msg(default_conf, mocker) -> None: telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = False - telegram.send_msg('test', bot) + telegram._send_msg('test', bot) assert not bot.method_calls bot.reset_mock() telegram._config['telegram']['enabled'] = True - telegram.send_msg('test', bot) + telegram._send_msg('test', bot) assert len(bot.method_calls) == 1 @@ -1113,7 +1113,7 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None: telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True - telegram.send_msg('test', bot) + telegram._send_msg('test', bot) # Bot should've tried to send it twice assert len(bot.method_calls) == 2 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5339ebc24..7a184eccc 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -57,7 +57,7 @@ def patch_RPCManager(mocker) -> MagicMock: :param mocker: mocker to patch RPCManager class :return: RPCManager.send_msg MagicMock to track if this method is called """ - mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) rpc_mock = mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) return rpc_mock From 4048859912c174822d1095ba94b3a2616bc11b73 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Jun 2018 04:52:50 +0200 Subject: [PATCH 041/239] rpc: remove tuple return madness --- freqtrade/rpc/rpc.py | 147 +++++++-------- freqtrade/rpc/telegram.py | 217 +++++++++++------------ freqtrade/tests/rpc/test_rpc.py | 139 ++++++--------- freqtrade/tests/rpc/test_rpc_manager.py | 4 +- freqtrade/tests/rpc/test_rpc_telegram.py | 8 +- 5 files changed, 223 insertions(+), 292 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 5ac78f2cb..2270214ab 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -5,7 +5,7 @@ import logging from abc import abstractmethod from datetime import datetime, timedelta, date from decimal import Decimal -from typing import Dict, Tuple, Any +from typing import Dict, Tuple, Any, List import arrow import sqlalchemy as sql @@ -20,6 +20,10 @@ from freqtrade.state import State logger = logging.getLogger(__name__) +class RPCException(Exception): + pass + + class RPC(object): """ RPC class can be used to have extra feature, like bot data, and access to DB data @@ -33,30 +37,29 @@ class RPC(object): self.freqtrade = freqtrade @abstractmethod - def cleanup(self) -> str: + def cleanup(self) -> None: """ Cleanup pending module resources """ @property @abstractmethod - def name(self) -> None: + def name(self) -> str: """ Returns the lowercase name of this module """ @abstractmethod def send_msg(self, msg: str) -> None: """ Sends a message to all registered rpc modules """ - def rpc_trade_status(self) -> Tuple[bool, Any]: + def _rpc_trade_status(self) -> List[str]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function - :return: """ # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self.freqtrade.state != State.RUNNING: - return True, '*Status:* `trader is not running`' + raise RPCException('*Status:* `trader is not running`') elif not trades: - return True, '*Status:* `no active trade`' + raise RPCException('*Status:* `no active trade`') else: result = [] for trade in trades: @@ -95,14 +98,14 @@ class RPC(object): ) if order else None, ) result.append(message) - return False, result + return result - def rpc_status_table(self) -> Tuple[bool, Any]: + def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self.freqtrade.state != State.RUNNING: - return True, '*Status:* `trader is not running`' + raise RPCException('*Status:* `trader is not running`') elif not trades: - return True, '*Status:* `no active order`' + raise RPCException('*Status:* `no active order`') else: trades_list = [] for trade in trades: @@ -118,20 +121,16 @@ class RPC(object): columns = ['ID', 'Pair', 'Since', 'Profit'] df_statuses = DataFrame.from_records(trades_list, columns=columns) df_statuses = df_statuses.set_index(columns[0]) - # The style used throughout is to return a tuple - # consisting of (error_occured?, result) - # Another approach would be to just return the - # result, or raise error - return False, df_statuses + return df_statuses - def rpc_daily_profit( + def _rpc_daily_profit( self, timescale: int, - stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: + stake_currency: str, fiat_display_currency: str) -> List[List[Any]]: today = datetime.utcnow().date() profit_days: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): - return True, '*Daily [n]:* `must be an integer greater than 0`' + raise RPCException('*Daily [n]:* `must be an integer greater than 0`') fiat = self.freqtrade.fiat_converter for day in range(0, timescale): @@ -148,7 +147,7 @@ class RPC(object): 'trades': len(trades) } - stats = [ + return [ [ key, '{value:.8f} {symbol}'.format( @@ -170,13 +169,10 @@ class RPC(object): ] for key, value in profit_days.items() ] - return False, stats - def rpc_trade_statistics( - self, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: - """ - :return: cumulative profit statistics. - """ + def _rpc_trade_statistics( + self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: + """ Returns cumulative profit statistics """ trades = Trade.query.order_by(Trade.id).all() profit_all_coin = [] @@ -214,7 +210,7 @@ class RPC(object): .order_by(sql.text('profit_sum DESC')).first() if not best_pair: - return True, '*Status:* `no closed trade`' + raise RPCException('*Status:* `no closed trade`') bp_pair, bp_rate = best_pair @@ -237,35 +233,29 @@ class RPC(object): fiat_display_currency ) num = float(len(durations) or 1) - return ( - False, - { - 'profit_closed_coin': profit_closed_coin, - 'profit_closed_percent': profit_closed_percent, - 'profit_closed_fiat': profit_closed_fiat, - 'profit_all_coin': profit_all_coin, - 'profit_all_percent': profit_all_percent, - 'profit_all_fiat': profit_all_fiat, - 'trade_count': len(trades), - 'first_trade_date': arrow.get(trades[0].open_date).humanize(), - 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), - 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], - 'best_pair': bp_pair, - 'best_rate': round(bp_rate * 100, 2) - } - ) + return { + 'profit_closed_coin': profit_closed_coin, + 'profit_closed_percent': profit_closed_percent, + 'profit_closed_fiat': profit_closed_fiat, + 'profit_all_coin': profit_all_coin, + 'profit_all_percent': profit_all_percent, + 'profit_all_fiat': profit_all_fiat, + 'trade_count': len(trades), + 'first_trade_date': arrow.get(trades[0].open_date).humanize(), + 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), + 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], + 'best_pair': bp_pair, + 'best_rate': round(bp_rate * 100, 2), + } - def rpc_balance(self, fiat_display_currency: str) -> Tuple[bool, Any]: - """ - :return: current account balance per crypto - """ + def _rpc_balance(self, fiat_display_currency: str) -> Tuple[List[Dict], float, str, float]: + """ Returns current account balance per crypto """ output = [] total = 0.0 for coin, balance in exchange.get_balances().items(): if not balance['total']: continue - rate = None if coin == 'BTC': rate = 1.0 else: @@ -285,32 +275,28 @@ class RPC(object): } ) if total == 0.0: - return True, '`All balances are zero.`' + raise RPCException('`All balances are zero.`') fiat = self.freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) - return False, (output, total, symbol, value) + return output, total, symbol, value - def rpc_start(self) -> Tuple[bool, str]: - """ - Handler for start. - """ + def _rpc_start(self) -> str: + """ Handler for start """ if self.freqtrade.state == State.RUNNING: - return True, '*Status:* `already running`' + return '*Status:* `already running`' self.freqtrade.state = State.RUNNING - return False, '`Starting trader ...`' + return '`Starting trader ...`' - def rpc_stop(self) -> Tuple[bool, str]: - """ - Handler for stop. - """ + def _rpc_stop(self) -> str: + """ Handler for stop """ if self.freqtrade.state == State.RUNNING: self.freqtrade.state = State.STOPPED - return False, '`Stopping trader ...`' + return '`Stopping trader ...`' - return True, '*Status:* `already stopped`' + return '*Status:* `already stopped`' def rpc_reload_conf(self) -> str: """ Handler for reload_conf. """ @@ -318,11 +304,10 @@ class RPC(object): return '*Status:* `Reloading config ...`' # FIX: no test for this!!!! - def rpc_forcesell(self, trade_id) -> Tuple[bool, Any]: + def _rpc_forcesell(self, trade_id) -> None: """ Handler for forcesell . Sells the given trade at current price - :return: error or None """ def _exec_forcesell(trade: Trade) -> None: # Check if there is there is an open order @@ -352,13 +337,13 @@ class RPC(object): # ---- EOF def _exec_forcesell ---- if self.freqtrade.state != State.RUNNING: - return True, '`trader is not running`' + raise RPCException('`trader is not running`') if trade_id == 'all': # Execute sell for all open orders for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): _exec_forcesell(trade) - return False, '' + return # Query for trade trade = Trade.query.filter( @@ -369,19 +354,18 @@ class RPC(object): ).first() if not trade: logger.warning('forcesell: Invalid argument received') - return True, 'Invalid argument.' + raise RPCException('Invalid argument.') _exec_forcesell(trade) Trade.session.flush() - return False, '' - def rpc_performance(self) -> Tuple[bool, Any]: + def _rpc_performance(self) -> List[Dict]: """ Handler for performance. Shows a performance statistic from finished trades """ if self.freqtrade.state != State.RUNNING: - return True, '`trader is not running`' + raise RPCException('`trader is not running`') pair_rates = Trade.session.query(Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum'), @@ -390,19 +374,14 @@ class RPC(object): .group_by(Trade.pair) \ .order_by(sql.text('profit_sum DESC')) \ .all() - trades = [] - for (pair, rate, count) in pair_rates: - trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) + return [ + {'pair': pair, 'profit': round(rate * 100, 2), 'count': count} + for pair, rate, count in pair_rates + ] - return False, trades - - def rpc_count(self) -> Tuple[bool, Any]: - """ - Returns the number of trades running - :return: None - """ + def _rpc_count(self) -> List[Trade]: + """ Returns the number of trades running """ if self.freqtrade.state != State.RUNNING: - return True, '`trader is not running`' + raise RPCException('`trader is not running`') - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - return False, trades + return Trade.query.filter(Trade.is_open.is_(True)).all() diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c00ba6a8e..f3cb65edc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -12,7 +12,7 @@ from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) @@ -151,13 +151,11 @@ class Telegram(RPC): self._status_table(bot, update) return - # Fetch open trade - (error, trades) = self.rpc_trade_status() - if error: - self._send_msg(trades, bot=bot) - else: - for trademsg in trades: - self._send_msg(trademsg, bot=bot) + try: + for trade_msg in self._rpc_trade_status(): + self._send_msg(trade_msg, bot=bot) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _status_table(self, bot: Bot, update: Update) -> None: @@ -168,15 +166,12 @@ class Telegram(RPC): :param update: message update :return: None """ - # Fetch open trade - (err, df_statuses) = self.rpc_status_table() - if err: - self._send_msg(df_statuses, bot=bot) - else: + try: + df_statuses = self._rpc_status_table() message = tabulate(df_statuses, headers='keys', tablefmt='simple') - message = "
{}
".format(message) - - self._send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg("
{}
".format(message), parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _daily(self, bot: Bot, update: Update) -> None: @@ -191,14 +186,12 @@ class Telegram(RPC): timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): timescale = 7 - (error, stats) = self.rpc_daily_profit( - timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] - ) - if error: - self._send_msg(stats, bot=bot) - else: + try: + stats = self._rpc_daily_profit( + timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) stats = tabulate(stats, headers=[ 'Day', @@ -207,11 +200,10 @@ class Telegram(RPC): ], tablefmt='simple') message = 'Daily Profit over the last {} days:\n
{}
'\ - .format( - timescale, - stats - ) + .format(timescale, stats) self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _profit(self, bot: Bot, update: Update) -> None: @@ -222,67 +214,65 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, stats) = self.rpc_trade_statistics( - self._config['stake_currency'], - self._config['fiat_display_currency'] - ) - if error: - self._send_msg(stats, bot=bot) - return + try: + stats = self._rpc_trade_statistics( + self._config['stake_currency'], + self._config['fiat_display_currency']) - # Message to display - markdown_msg = "*ROI:* Close trades\n" \ - "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ - "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ - "*ROI:* All trades\n" \ - "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ - "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ - "*Total Trade Count:* `{trade_count}`\n" \ - "*First Trade opened:* `{first_trade_date}`\n" \ - "*Latest Trade opened:* `{latest_trade_date}`\n" \ - "*Avg. Duration:* `{avg_duration}`\n" \ - "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ - .format( - coin=self._config['stake_currency'], - fiat=self._config['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) - self._send_msg(markdown_msg, bot=bot) + # Message to display + markdown_msg = "*ROI:* Close trades\n" \ + "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ + "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ + "*ROI:* All trades\n" \ + "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ + "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ + "*Total Trade Count:* `{trade_count}`\n" \ + "*First Trade opened:* `{first_trade_date}`\n" \ + "*Latest Trade opened:* `{latest_trade_date}`\n" \ + "*Avg. Duration:* `{avg_duration}`\n" \ + "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ + .format( + coin=self._config['stake_currency'], + fiat=self._config['fiat_display_currency'], + profit_closed_coin=stats['profit_closed_coin'], + profit_closed_percent=stats['profit_closed_percent'], + profit_closed_fiat=stats['profit_closed_fiat'], + profit_all_coin=stats['profit_all_coin'], + profit_all_percent=stats['profit_all_percent'], + profit_all_fiat=stats['profit_all_fiat'], + trade_count=stats['trade_count'], + first_trade_date=stats['first_trade_date'], + latest_trade_date=stats['latest_trade_date'], + avg_duration=stats['avg_duration'], + best_pair=stats['best_pair'], + best_rate=stats['best_rate'] + ) + self._send_msg(markdown_msg, bot=bot) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _balance(self, bot: Bot, update: Update) -> None: """ Handler for /balance """ - (error, result) = self.rpc_balance(self._config['fiat_display_currency']) - if error: - self._send_msg('`All balances are zero.`') - return + try: + currencys, total, symbol, value = \ + self._rpc_balance(self._config['fiat_display_currency']) + output = '' + for currency in currencys: + output += "*{currency}:*\n" \ + "\t`Available: {available: .8f}`\n" \ + "\t`Balance: {balance: .8f}`\n" \ + "\t`Pending: {pending: .8f}`\n" \ + "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) - (currencys, total, symbol, value) = result - output = '' - for currency in currencys: - output += "*{currency}:*\n" \ - "\t`Available: {available: .8f}`\n" \ - "\t`Balance: {balance: .8f}`\n" \ - "\t`Pending: {pending: .8f}`\n" \ - "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) - - output += "\n*Estimated Value*:\n" \ - "\t`BTC: {0: .8f}`\n" \ - "\t`{1}: {2: .2f}`\n".format(total, symbol, value) - self._send_msg(output) + output += "\n*Estimated Value*:\n" \ + "\t`BTC: {0: .8f}`\n" \ + "\t`{1}: {2: .2f}`\n".format(total, symbol, value) + self._send_msg(output, bot=bot) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _start(self, bot: Bot, update: Update) -> None: @@ -293,9 +283,8 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, msg) = self.rpc_start() - if error: - self._send_msg(msg, bot=bot) + msg = self._rpc_start() + self._send_msg(msg, bot=bot) @authorized_only def _stop(self, bot: Bot, update: Update) -> None: @@ -306,7 +295,7 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, msg) = self.rpc_stop() + msg = self._rpc_stop() self._send_msg(msg, bot=bot) @authorized_only @@ -332,10 +321,10 @@ class Telegram(RPC): """ trade_id = update.message.text.replace('/forcesell', '').strip() - (error, message) = self.rpc_forcesell(trade_id) - if error: - self._send_msg(message, bot=bot) - return + try: + self._rpc_forcesell(trade_id) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _performance(self, bot: Bot, update: Update) -> None: @@ -346,19 +335,18 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, trades) = self.rpc_performance() - if error: - self._send_msg(trades, bot=bot) - return - - stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( - index=i + 1, - pair=trade['pair'], - profit=trade['profit'], - count=trade['count'] - ) for i, trade in enumerate(trades)) - message = 'Performance:\n{}'.format(stats) - self._send_msg(message, parse_mode=ParseMode.HTML) + try: + trades = self._rpc_performance() + stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( + index=i + 1, + pair=trade['pair'], + profit=trade['profit'], + count=trade['count'] + ) for i, trade in enumerate(trades)) + message = 'Performance:\n{}'.format(stats) + self._send_msg(message, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _count(self, bot: Bot, update: Update) -> None: @@ -369,19 +357,18 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, trades) = self.rpc_count() - if error: - self._send_msg(trades, bot=bot) - return - - message = tabulate({ - 'current': [len(trades)], - 'max': [self._config['max_open_trades']], - 'total stake': [sum((trade.open_rate * trade.amount) for trade in trades)] - }, headers=['current', 'max', 'total stake'], tablefmt='simple') - message = "
{}
".format(message) - logger.debug(message) - self._send_msg(message, parse_mode=ParseMode.HTML) + try: + trades = self._rpc_count() + message = tabulate({ + 'current': [len(trades)], + 'max': [self._config['max_open_trades']], + 'total stake': [sum((trade.open_rate * trade.amount) for trade in trades)] + }, headers=['current', 'max', 'total stake'], tablefmt='simple') + message = "
{}
".format(message) + logger.debug(message) + self._send_msg(message, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _help(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 5cdd22c7a..5d7b56bc5 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -7,9 +7,11 @@ Unit test file for rpc/rpc.py from datetime import datetime from unittest.mock import MagicMock +import pytest + from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc.rpc import RPC, RPCException from freqtrade.state import State from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap @@ -41,19 +43,16 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, result) = rpc.rpc_trade_status() - assert error - assert 'trader is not running' in result + with pytest.raises(RPCException, match=r'.*trader is not running*'): + rpc._rpc_trade_status() freqtradebot.state = State.RUNNING - (error, result) = rpc.rpc_trade_status() - assert error - assert 'no active trade' in result + with pytest.raises(RPCException, match=r'.*no active trade*'): + rpc._rpc_trade_status() freqtradebot.create_trade() - (error, result) = rpc.rpc_trade_status() - assert not error - trade = result[0] + trades = rpc._rpc_trade_status() + trade = trades[0] result_message = [ '*Trade ID:* `1`\n' @@ -68,7 +67,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: '*Current Profit:* `-0.59%`\n' '*Open Order:* `(limit buy rem=0.00000000)`' ] - assert result == result_message + assert trades == result_message assert trade.find('[ETH/BTC]') >= 0 @@ -90,17 +89,15 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, result) = rpc.rpc_status_table() - assert error - assert '*Status:* `trader is not running`' in result + with pytest.raises(RPCException, match=r'.*\*Status:\* `trader is not running``*'): + rpc._rpc_status_table() freqtradebot.state = State.RUNNING - (error, result) = rpc.rpc_status_table() - assert error - assert '*Status:* `no active order`' in result + with pytest.raises(RPCException, match=r'.*\*Status:\* `no active order`*'): + rpc._rpc_status_table() freqtradebot.create_trade() - (error, result) = rpc.rpc_status_table() + result = rpc._rpc_status_table() assert 'just now' in result['Since'].all() assert 'ETH/BTC' in result['Pair'].all() assert '-0.59%' in result['Profit'].all() @@ -140,8 +137,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, # Try valid data update.message.text = '/daily 2' - (error, days) = rpc.rpc_daily_profit(7, stake_currency, fiat_display_currency) - assert not error + days = rpc._rpc_daily_profit(7, stake_currency, fiat_display_currency) assert len(days) == 7 for day in days: # [datetime.date(2018, 1, 11), '0.00000000 BTC', '0.000 USD'] @@ -154,9 +150,8 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, assert str(days[0][0]) == str(datetime.utcnow().date()) # Try invalid data - (error, days) = rpc.rpc_daily_profit(0, stake_currency, fiat_display_currency) - assert error - assert days.find('must be an integer greater than 0') >= 0 + with pytest.raises(RPCException, match=r'.*must be an integer greater than 0*'): + rpc._rpc_daily_profit(0, stake_currency, fiat_display_currency) def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, @@ -184,9 +179,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, rpc = RPC(freqtradebot) - (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) - assert error - assert stats.find('no closed trade') >= 0 + with pytest.raises(RPCException, match=r'.*no closed trade*'): + rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) # Create some test data freqtradebot.create_trade() @@ -219,8 +213,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, trade.close_date = datetime.utcnow() trade.is_open = False - (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) - assert not error + stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) @@ -281,8 +274,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, for trade in Trade.query.order_by(Trade.id).all(): trade.open_rate = None - (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) - assert not error + stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert prec_satoshi(stats['profit_closed_coin'], 0) assert prec_satoshi(stats['profit_closed_percent'], 0) assert prec_satoshi(stats['profit_closed_fiat'], 0) @@ -330,18 +322,16 @@ def test_rpc_balance_handle(default_conf, mocker): freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) - (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency']) - assert not error - (trade, x, y, z) = res - assert prec_satoshi(x, 12) - assert prec_satoshi(z, 180000) - assert 'USD' in y - assert len(trade) == 1 - assert 'BTC' in trade[0]['currency'] - assert prec_satoshi(trade[0]['available'], 10) - assert prec_satoshi(trade[0]['balance'], 12) - assert prec_satoshi(trade[0]['pending'], 2) - assert prec_satoshi(trade[0]['est_btc'], 12) + output, total, symbol, value = rpc._rpc_balance(default_conf['fiat_display_currency']) + assert prec_satoshi(total, 12) + assert prec_satoshi(value, 180000) + assert 'USD' in symbol + assert len(output) == 1 + assert 'BTC' in output[0]['currency'] + assert prec_satoshi(output[0]['available'], 10) + assert prec_satoshi(output[0]['balance'], 12) + assert prec_satoshi(output[0]['pending'], 2) + assert prec_satoshi(output[0]['est_btc'], 12) def test_rpc_start(mocker, default_conf) -> None: @@ -361,13 +351,11 @@ def test_rpc_start(mocker, default_conf) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, result) = rpc.rpc_start() - assert not error + result = rpc._rpc_start() assert '`Starting trader ...`' in result assert freqtradebot.state == State.RUNNING - (error, result) = rpc.rpc_start() - assert error + result = rpc._rpc_start() assert '*Status:* `already running`' in result assert freqtradebot.state == State.RUNNING @@ -389,13 +377,11 @@ def test_rpc_stop(mocker, default_conf) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING - (error, result) = rpc.rpc_stop() - assert not error + result = rpc._rpc_stop() assert '`Stopping trader ...`' in result assert freqtradebot.state == State.STOPPED - (error, result) = rpc.rpc_stop() - assert error + result = rpc._rpc_stop() assert '*Status:* `already stopped`' in result assert freqtradebot.state == State.STOPPED @@ -428,36 +414,26 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' + with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + rpc._rpc_forcesell(None) freqtradebot.state = State.RUNNING - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == 'Invalid argument.' + with pytest.raises(RPCException, match=r'.*Invalid argument.*'): + rpc._rpc_forcesell(None) - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' + rpc._rpc_forcesell('all') freqtradebot.create_trade() - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' + rpc._rpc_forcesell('all') - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' + rpc._rpc_forcesell('1') freqtradebot.state = State.STOPPED - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' + with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + rpc._rpc_forcesell(None) - (error, res) = rpc.rpc_forcesell('all') - assert error - assert res == '`trader is not running`' + with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + rpc._rpc_forcesell('all') freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 @@ -475,9 +451,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called # and trade amount is updated - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' + rpc._rpc_forcesell('1') assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount @@ -495,9 +469,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: } ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called - (error, res) = rpc.rpc_forcesell('2') - assert not error - assert res == '' + rpc._rpc_forcesell('2') assert cancel_order_mock.call_count == 2 assert trade.amount == amount @@ -511,9 +483,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'side': 'sell' } ) - (error, res) = rpc.rpc_forcesell('3') - assert not error - assert res == '' + rpc._rpc_forcesell('3') # status quo, no exchange calls assert cancel_order_mock.call_count == 2 @@ -550,8 +520,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, trade.close_date = datetime.utcnow() trade.is_open = False - (error, res) = rpc.rpc_performance() - assert not error + res = rpc._rpc_performance() assert len(res) == 1 assert res[0]['pair'] == 'ETH/BTC' assert res[0]['count'] == 1 @@ -576,14 +545,12 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) - (error, trades) = rpc.rpc_count() + trades = rpc._rpc_count() nb_trades = len(trades) - assert not error assert nb_trades == 0 # Create some test data freqtradebot.create_trade() - (error, trades) = rpc.rpc_count() + trades = rpc._rpc_count() nb_trades = len(trades) - assert not error assert nb_trades == 1 diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 6c073a251..805424d26 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -104,7 +104,7 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: rpc_manager = RPCManager(freqtradebot) rpc_manager.send_msg('test') - assert log_has('test', caplog.record_tuples) + assert log_has('Sending rpc message: test', caplog.record_tuples) assert telegram_mock.call_count == 0 @@ -119,5 +119,5 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: rpc_manager = RPCManager(freqtradebot) rpc_manager.send_msg('test') - assert log_has('test', caplog.record_tuples) + assert log_has('Sending rpc message: test', caplog.record_tuples) assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 47ccf4243..87e884110 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -60,9 +60,7 @@ def test__init__(default_conf, mocker) -> None: def test_init(default_conf, mocker, caplog) -> None: - """ - Test _init() method - """ + """ Test _init() method """ start_polling = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock(return_value=start_polling)) @@ -256,7 +254,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), + _rpc_trade_status=MagicMock(return_value=[1, 2, 3]), _status_table=status_table, _send_msg=msg_mock ) @@ -667,7 +665,7 @@ def test_start_handle(default_conf, update, mocker) -> None: assert freqtradebot.state == State.STOPPED telegram._start(bot=MagicMock(), update=update) assert freqtradebot.state == State.RUNNING - assert msg_mock.call_count == 0 + assert msg_mock.call_count == 1 def test_start_handle_already_running(default_conf, update, mocker) -> None: From 3787dad212f085deabe276aab0085e9e3e11988f Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Jun 2018 23:50:38 +0200 Subject: [PATCH 042/239] don't import python-telegram-bot at runtime if disabled in config --- freqtrade/rpc/rpc_manager.py | 2 +- freqtrade/rpc/telegram.py | 2 ++ freqtrade/tests/rpc/test_rpc.py | 22 +++++++++++----------- freqtrade/tests/test_freqtradebot.py | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index ce01b78a3..a2699c0e8 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -5,7 +5,6 @@ import logging from typing import List from freqtrade.rpc.rpc import RPC -from freqtrade.rpc.telegram import Telegram logger = logging.getLogger(__name__) @@ -21,6 +20,7 @@ class RPCManager(object): # Enable telegram if freqtrade.config['telegram'].get('enabled', False): logger.info('Enabling rpc.telegram ...') + from freqtrade.rpc.telegram import Telegram self.registered_modules.append(Telegram(freqtrade)) def cleanup(self) -> None: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f3cb65edc..08b45ffe4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -16,6 +16,8 @@ from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) +logger.debug('Included module rpc.telegram ...') + def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Callable[..., Any]: """ diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 5d7b56bc5..b49b7fdcb 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -31,7 +31,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -77,7 +77,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -110,7 +110,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -165,7 +165,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, ticker=MagicMock(return_value={'price_usd': 15000.0}), ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -241,7 +241,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, ticker=MagicMock(return_value={'price_usd': 15000.0}), ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -312,7 +312,7 @@ def test_rpc_balance_handle(default_conf, mocker): ticker=MagicMock(return_value={'price_usd': 15000.0}), ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -340,7 +340,7 @@ def test_rpc_start(mocker, default_conf) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -366,7 +366,7 @@ def test_rpc_stop(mocker, default_conf) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -392,7 +392,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) cancel_order_mock = MagicMock() mocker.patch.multiple( @@ -495,7 +495,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -533,7 +533,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 7a184eccc..1d272428e 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -57,7 +57,7 @@ def patch_RPCManager(mocker) -> MagicMock: :param mocker: mocker to patch RPCManager class :return: RPCManager.send_msg MagicMock to track if this method is called """ - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) rpc_mock = mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) return rpc_mock From 34e10a145c946eabc8be69050408f5c3e7440dbc Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 00:15:04 +0200 Subject: [PATCH 043/239] remove Telegram.is_enabled() because RPCManager manages lifecycles --- freqtrade/rpc/telegram.py | 17 ------- freqtrade/tests/rpc/test_rpc_telegram.py | 58 ++---------------------- 2 files changed, 4 insertions(+), 71 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 08b45ffe4..701c23a51 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -80,12 +80,7 @@ class Telegram(RPC): Initializes this module with the given config, registers all known command handlers and starts polling for message updates - :param config: config to use - :return: None """ - if not self.is_enabled(): - return - self._updater = Updater(token=self._config['telegram']['token'], workers=0) # Register command handler and start telegram message polling @@ -121,21 +116,12 @@ class Telegram(RPC): Stops all running telegram threads. :return: None """ - if not self.is_enabled(): - return - self._updater.stop() def send_msg(self, msg: str) -> None: """ Send a message to telegram channel """ self._send_msg(msg) - def is_enabled(self) -> bool: - """ - Returns True if the telegram module is activated, False otherwise - """ - return bool(self._config.get('telegram', {}).get('enabled', False)) - @authorized_only def _status(self, bot: Bot, update: Update) -> None: """ @@ -418,9 +404,6 @@ class Telegram(RPC): :param parse_mode: telegram parse mode :return: None """ - if not self.is_enabled(): - return - bot = bot or self._updater.bot keyboard = [['/daily', '/profit', '/balance'], diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 87e884110..68afa7587 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -32,6 +32,9 @@ class DummyCls(Telegram): super().__init__(freqtrade) self.state = {'called': False} + def _init(self): + pass + @authorized_only def dummy_handler(self, *args, **kwargs) -> None: """ @@ -78,21 +81,6 @@ def test_init(default_conf, mocker, caplog) -> None: assert log_has(message_str, caplog.record_tuples) -def test_init_disabled(default_conf, mocker, caplog) -> None: - """ - Test _init() method when Telegram is disabled - """ - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - Telegram(get_patched_freqtradebot(mocker, conf)) - - message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ - "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \ - "['count'], ['help'], ['version']]" - - assert not log_has(message_str, caplog.record_tuples) - - def test_cleanup(default_conf, mocker) -> None: """ Test cleanup() method @@ -101,44 +89,11 @@ def test_cleanup(default_conf, mocker) -> None: updater_mock.stop = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock) - # not enabled - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - telegram = Telegram(get_patched_freqtradebot(mocker, conf)) - telegram.cleanup() - assert telegram._updater is None - assert updater_mock.call_count == 0 - assert not hasattr(telegram._updater, 'stop') - assert updater_mock.stop.call_count == 0 - - # enabled - conf['telegram']['enabled'] = True - telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) telegram.cleanup() assert telegram._updater.stop.call_count == 1 -def test_is_enabled(default_conf, mocker) -> None: - """ - Test is_enabled() method - """ - mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) - - telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) - assert telegram.is_enabled() - - -def test_is_not_enabled(default_conf, mocker) -> None: - """ - Test is_enabled() method - """ - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - telegram = Telegram(get_patched_freqtradebot(mocker, conf)) - - assert not telegram.is_enabled() - - def test_authorized_only(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are authorized @@ -1087,11 +1042,6 @@ def test_send_msg(default_conf, mocker) -> None: freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) - telegram._config['telegram']['enabled'] = False - telegram._send_msg('test', bot) - assert not bot.method_calls - bot.reset_mock() - telegram._config['telegram']['enabled'] = True telegram._send_msg('test', bot) assert len(bot.method_calls) == 1 From 6c1bb7983bef16659d6c4a0ad5a58400909d6e3c Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 00:20:10 +0200 Subject: [PATCH 044/239] rpc: make freqtrade a private variable --- freqtrade/rpc/rpc.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2270214ab..0d81ae8ae 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -34,7 +34,7 @@ class RPC(object): :param freqtrade: Instance of a freqtrade bot :return: None """ - self.freqtrade = freqtrade + self._freqtrade = freqtrade @abstractmethod def cleanup(self) -> None: @@ -56,7 +56,7 @@ class RPC(object): """ # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('*Status:* `trader is not running`') elif not trades: raise RPCException('*Status:* `no active trade`') @@ -102,7 +102,7 @@ class RPC(object): def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('*Status:* `trader is not running`') elif not trades: raise RPCException('*Status:* `no active order`') @@ -132,7 +132,7 @@ class RPC(object): if not (isinstance(timescale, int) and timescale > 0): raise RPCException('*Daily [n]:* `must be an integer greater than 0`') - fiat = self.freqtrade.fiat_converter + fiat = self._freqtrade.fiat_converter for day in range(0, timescale): profitday = today - timedelta(days=day) trades = Trade.query \ @@ -216,7 +216,7 @@ class RPC(object): # FIX: we want to keep fiatconverter in a state/environment, # doing this will utilize its caching functionallity, instead we reinitialize it here - fiat = self.freqtrade.fiat_converter + fiat = self._freqtrade.fiat_converter # Prepare data to display profit_closed_coin = round(sum(profit_closed_coin), 8) profit_closed_percent = round(nan_to_num(mean(profit_closed_percent)) * 100, 2) @@ -277,23 +277,23 @@ class RPC(object): if total == 0.0: raise RPCException('`All balances are zero.`') - fiat = self.freqtrade.fiat_converter + fiat = self._freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) return output, total, symbol, value def _rpc_start(self) -> str: """ Handler for start """ - if self.freqtrade.state == State.RUNNING: + if self._freqtrade.state == State.RUNNING: return '*Status:* `already running`' - self.freqtrade.state = State.RUNNING + self._freqtrade.state = State.RUNNING return '`Starting trader ...`' def _rpc_stop(self) -> str: """ Handler for stop """ - if self.freqtrade.state == State.RUNNING: - self.freqtrade.state = State.STOPPED + if self._freqtrade.state == State.RUNNING: + self._freqtrade.state = State.STOPPED return '`Stopping trader ...`' return '*Status:* `already stopped`' @@ -333,10 +333,10 @@ class RPC(object): # Get current rate and execute sell current_rate = exchange.get_ticker(trade.pair, False)['bid'] - self.freqtrade.execute_sell(trade, current_rate) + self._freqtrade.execute_sell(trade, current_rate) # ---- EOF def _exec_forcesell ---- - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('`trader is not running`') if trade_id == 'all': @@ -364,7 +364,7 @@ class RPC(object): Handler for performance. Shows a performance statistic from finished trades """ - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('`trader is not running`') pair_rates = Trade.session.query(Trade.pair, @@ -381,7 +381,7 @@ class RPC(object): def _rpc_count(self) -> List[Trade]: """ Returns the number of trades running """ - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('`trader is not running`') return Trade.query.filter(Trade.is_open.is_(True)).all() From 83eb7a0a9d7b108f2f1626f759f1d9d3acc0b5cb Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 00:58:24 +0200 Subject: [PATCH 045/239] adjust logging a bit and add some comments --- freqtrade/rpc/rpc.py | 6 ++++++ freqtrade/rpc/rpc_manager.py | 3 ++- freqtrade/rpc/telegram.py | 12 +++--------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0d81ae8ae..10bbc854a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -21,6 +21,12 @@ logger = logging.getLogger(__name__) class RPCException(Exception): + """ + Should be raised with a rpc-formatted message in an _rpc_* method + if the required state is wrong, i.e.: + + raise RPCException('*Status:* `no active trade`') + """ pass diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index a2699c0e8..370d18176 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -25,8 +25,9 @@ class RPCManager(object): def cleanup(self) -> None: """ Stops all enabled rpc modules """ + logger.info('Cleaning up rpc modules ...') for mod in self.registered_modules: - logger.info('Cleaning up rpc.%s ...', mod.name) + logger.debug('Cleaning up rpc.%s ...', mod.name) mod.cleanup() self.registered_modules = [] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 701c23a51..e9091eb2b 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -26,9 +26,7 @@ def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Call :return: decorated function """ def wrapper(self, *args, **kwargs): - """ - Decorator logic - """ + """ Decorator logic """ update = kwargs.get('update') or args[1] # Reject unauthorized messages @@ -55,9 +53,7 @@ def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Call class Telegram(RPC): - """ - Telegram, this class send messages to Telegram - """ + """ This class handles all telegram communication """ @property def name(self) -> str: @@ -241,9 +237,7 @@ class Telegram(RPC): @authorized_only def _balance(self, bot: Bot, update: Update) -> None: - """ - Handler for /balance - """ + """ Handler for /balance """ try: currencys, total, symbol, value = \ self._rpc_balance(self._config['fiat_display_currency']) From e14c9e2090c9175c93ca8eecfa8c216b8bebcb7e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 13:30:48 +0200 Subject: [PATCH 046/239] fix potential cleanup issue --- freqtrade/rpc/rpc_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 370d18176..252bbcdd8 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -26,11 +26,11 @@ class RPCManager(object): def cleanup(self) -> None: """ Stops all enabled rpc modules """ logger.info('Cleaning up rpc modules ...') - for mod in self.registered_modules: + while self.registered_modules: + mod = self.registered_modules.pop() logger.debug('Cleaning up rpc.%s ...', mod.name) mod.cleanup() - - self.registered_modules = [] + del mod def send_msg(self, msg: str) -> None: """ From 92b0cbdc19ad3664cac637e9aa1c936760d48202 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:07 +0200 Subject: [PATCH 047/239] Update ccxt from 1.14.177 to 1.14.186 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2272d47b8..8414ba7d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.177 +ccxt==1.14.186 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From f404e0f5b39e14add3443e088956e5dc5b5da8ba Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:08 +0200 Subject: [PATCH 048/239] Update requests from 2.18.4 to 2.19.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8414ba7d9..d1dd93dc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 -requests==2.18.4 +requests==2.19.0 urllib3==1.22 wrapt==1.10.11 pandas==0.23.0 From 038acd3f5eca3de209547f8d0c035887c544e197 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:09 +0200 Subject: [PATCH 049/239] Update pandas from 0.23.0 to 0.23.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d1dd93dc9..e5ac314d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==2.1.0 requests==2.19.0 urllib3==1.22 wrapt==1.10.11 -pandas==0.23.0 +pandas==0.23.1 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 From 875408215bd87d364b0a09701590707fc4b18f57 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:11 +0200 Subject: [PATCH 050/239] Update numpy from 1.14.4 to 1.14.5 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e5ac314d1..491297708 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.1 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.14.4 +numpy==1.14.5 TA-Lib==0.4.17 pytest==3.6.1 pytest-mock==1.10.0 From 46080f516897d9b005ffee8a540dccd431a3bc07 Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 13 Jun 2018 15:29:27 +0200 Subject: [PATCH 051/239] define _rpc_reload_conf as private method --- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/rpc/telegram.py | 4 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 10bbc854a..34802f920 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -304,9 +304,9 @@ class RPC(object): return '*Status:* `already stopped`' - def rpc_reload_conf(self) -> str: + def _rpc_reload_conf(self) -> str: """ Handler for reload_conf. """ - self.freqtrade.state = State.RELOAD_CONF + self._freqtrade.state = State.RELOAD_CONF return '*Status:* `Reloading config ...`' # FIX: no test for this!!!! diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e9091eb2b..4dd23971b 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -289,8 +289,8 @@ class Telegram(RPC): :param update: message update :return: None """ - msg = self.rpc_reload_conf() - self.send_msg(msg, bot=bot) + msg = self._rpc_reload_conf() + self._send_msg(msg, bot=bot) @authorized_only def _forcesell(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 68afa7587..f022c09e4 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -706,7 +706,7 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) From 61f92b7460b1781c0a17a149b4fdbdf1f297f181 Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 31 Oct 2017 21:57:58 +0200 Subject: [PATCH 052/239] bugfix --- freqtrade/vendor/qtpylib/indicators.py | 50 ++++++++++---------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index ee1f14e1f..c4b955626 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -248,45 +248,34 @@ def crossed_below(series1, series2): def rolling_std(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods - try: - if min_periods == window: - return numpy_rolling_std(series, window, True) - else: - try: - return series.rolling(window=window, min_periods=min_periods).std() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).std() - except BaseException: - return pd.rolling_std(series, window=window, min_periods=min_periods) - + if min_periods == window and len(series) > window: + return numpy_rolling_std(series, window, True) + else: + try: + return series.rolling(window=window, min_periods=min_periods).std() + except BaseException: + return pd.Series(series).rolling(window=window, min_periods=min_periods).std() # --------------------------------------------- def rolling_mean(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods - try: - if min_periods == window: - return numpy_rolling_mean(series, window, True) - else: - try: - return series.rolling(window=window, min_periods=min_periods).mean() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).mean() - except BaseException: - return pd.rolling_mean(series, window=window, min_periods=min_periods) - + if min_periods == window and len(series) > window: + return numpy_rolling_mean(series, window, True) + else: + try: + return series.rolling(window=window, min_periods=min_periods).mean() + except BaseException: + return pd.Series(series).rolling(window=window, min_periods=min_periods).mean() # --------------------------------------------- def rolling_min(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: - try: - return series.rolling(window=window, min_periods=min_periods).min() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).min() + return series.rolling(window=window, min_periods=min_periods).min() except BaseException: - return pd.rolling_min(series, window=window, min_periods=min_periods) + return pd.Series(series).rolling(window=window, min_periods=min_periods).min() # --------------------------------------------- @@ -294,12 +283,9 @@ def rolling_min(series, window=14, min_periods=None): def rolling_max(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: - try: - return series.rolling(window=window, min_periods=min_periods).min() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).min() + return series.rolling(window=window, min_periods=min_periods).min() except BaseException: - return pd.rolling_min(series, window=window, min_periods=min_periods) + return pd.Series(series).rolling(window=window, min_periods=min_periods).min() # --------------------------------------------- From e6e5c5daf02a88fa875f5445c5fece13aaf109ef Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 31 Oct 2017 21:58:03 +0200 Subject: [PATCH 053/239] added zlma --- freqtrade/vendor/qtpylib/indicators.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index c4b955626..22fac4e2a 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -552,6 +552,26 @@ def stoch(df, window=14, d=3, k=3, fast=False): return pd.DataFrame(index=df.index, data=data) +# --------------------------------------------- +def zlma(series, window=20, kind="ema"): + """ + John Ehlers' Zero lag (exponential) moving average + https://en.wikipedia.org/wiki/Zero_lag_exponential_moving_average + """ + lag = (window - 1) // 2 + series = 2 * series - series.shift(lag) + if kind in ['ewm', 'ema']: + return ema(series, lag) + elif kind == "hma": + return hma(series, lag) + return sma(series, lag) + +def zlema(series, window): + return zlma(series, window, kind="ema") +def zlsma(series, window): + return zlma(series, window, kind="sma") +def zlhma(series, window): + return zlma(series, window, kind="hma") # --------------------------------------------- From 6edb25f5c2f083060e68e274f23ddcdb25d23e03 Mon Sep 17 00:00:00 2001 From: ran Date: Fri, 12 Jan 2018 09:27:52 +0200 Subject: [PATCH 054/239] fixed heikenashi calculation --- freqtrade/vendor/qtpylib/indicators.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 22fac4e2a..9830e08fe 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -110,10 +110,13 @@ def heikinashi(bars): bars = bars.copy() bars['ha_close'] = (bars['open'] + bars['high'] + bars['low'] + bars['close']) / 4 + bars['ha_open'] = (bars['open'].shift(1) + bars['close'].shift(1)) / 2 bars.loc[:1, 'ha_open'] = bars['open'].values[0] - bars.loc[1:, 'ha_open'] = ( - (bars['ha_open'].shift(1) + bars['ha_close'].shift(1)) / 2)[1:] + for x in range(2): + bars.loc[1:, 'ha_open'] = ( + (bars['ha_open'].shift(1) + bars['ha_close'].shift(1)) / 2)[1:] + bars['ha_high'] = bars.loc[:, ['high', 'ha_open', 'ha_close']].max(axis=1) bars['ha_low'] = bars.loc[:, ['low', 'ha_open', 'ha_close']].min(axis=1) From d684ff5715271a2ef52bf2cd9b85783b5077d69f Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 13 Jun 2018 16:20:13 +0200 Subject: [PATCH 055/239] drop zlma implementation --- freqtrade/vendor/qtpylib/indicators.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 9830e08fe..e68932998 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -261,6 +261,7 @@ def rolling_std(series, window=200, min_periods=None): # --------------------------------------------- + def rolling_mean(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods if min_periods == window and len(series) > window: @@ -273,6 +274,7 @@ def rolling_mean(series, window=200, min_periods=None): # --------------------------------------------- + def rolling_min(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: @@ -556,27 +558,7 @@ def stoch(df, window=14, d=3, k=3, fast=False): return pd.DataFrame(index=df.index, data=data) # --------------------------------------------- -def zlma(series, window=20, kind="ema"): - """ - John Ehlers' Zero lag (exponential) moving average - https://en.wikipedia.org/wiki/Zero_lag_exponential_moving_average - """ - lag = (window - 1) // 2 - series = 2 * series - series.shift(lag) - if kind in ['ewm', 'ema']: - return ema(series, lag) - elif kind == "hma": - return hma(series, lag) - return sma(series, lag) -def zlema(series, window): - return zlma(series, window, kind="ema") -def zlsma(series, window): - return zlma(series, window, kind="sma") -def zlhma(series, window): - return zlma(series, window, kind="hma") - -# --------------------------------------------- def zscore(bars, window=20, stds=1, col='close'): """ get zscore of price """ From e600be4f568b8f3712e01a9381553ac618507611 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 19:43:33 +0200 Subject: [PATCH 056/239] Reduce force-sell verbosity --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5df4fb28a..dbbdd4b80 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -176,8 +176,8 @@ class Backtesting(object): close_index=sell_row.Index, open_at_end=True ) - logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, - btr.profit_percent, btr.profit_abs) + logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, + btr.profit_percent, btr.profit_abs) return btr return None From c0289ad8449b22800cc2272ac6180d486eca621f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 19:53:12 +0200 Subject: [PATCH 057/239] use list comprehension to build list --- freqtrade/optimize/backtesting.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index dbbdd4b80..df68c1f31 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -114,14 +114,11 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - records = [] - print(results) - for index, trade_entry in results.iterrows(): - pass - records.append((trade_entry.pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - trade_entry.open_index - 1, trade_entry.trade_duration)) + records = [(trade_entry.pair, trade_entry.profit_percent, + trade_entry.open_time.timestamp(), + trade_entry.close_time.timestamp(), + trade_entry.open_index - 1, trade_entry.trade_duration) + for index, trade_entry in results.iterrows()] if records: logger.info('Dumping backtest results to %s', recordfilename) From ea805a8fb7e5f12e02d781955d86bd639f48424e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 14 Jun 2018 14:22:06 +0200 Subject: [PATCH 058/239] Update ccxt from 1.14.186 to 1.14.196 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 491297708..b1821f785 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.186 +ccxt==1.14.196 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 5c3e37412e2d9976252f41a9ebb84f65148ae42e Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 14 Jun 2018 21:20:16 +0200 Subject: [PATCH 059/239] update docs --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 127b4ee20..1efb46b43 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -21,7 +21,7 @@ If the 5 min and 1 min ticker for the crypto-currencies to test is not already in the `testdata` folder, backtesting will download them automatically. Testdata files will not be updated until you specify it. -The result of backtesting will confirm you if your bot as more chance to make a profit than a loss. +The result of backtesting will confirm you if your bot has better odds of making a profit than a loss. The backtesting is very easy with freqtrade. From 1e208e39b08f2dd1eb0016cda01305a4b42aaae5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:07 +0200 Subject: [PATCH 060/239] Update ccxt from 1.14.196 to 1.14.198 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b1821f785..bd5b3b10a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.196 +ccxt==1.14.198 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From e8fd11d6cecf825188a1bc2f2b6210dd8c1ffdba Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:08 +0200 Subject: [PATCH 061/239] Update requests from 2.19.0 to 2.19.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd5b3b10a..e8240fd97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 -requests==2.19.0 +requests==2.19.1 urllib3==1.22 wrapt==1.10.11 pandas==0.23.1 From a8d25266f962ac9989d570017b42c297b7325774 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:07 +0200 Subject: [PATCH 062/239] Update ccxt from 1.14.196 to 1.14.198 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b1821f785..bd5b3b10a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.196 +ccxt==1.14.198 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From af16830a38e10a9ec0395d2c1bb38239646ebf13 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:08 +0200 Subject: [PATCH 063/239] Update requests from 2.19.0 to 2.19.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd5b3b10a..e8240fd97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 -requests==2.19.0 +requests==2.19.1 urllib3==1.22 wrapt==1.10.11 pandas==0.23.1 From c1f8f641e6a696b513a8eb81d9e246d86b861df0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 15 Jun 2018 10:45:19 +0300 Subject: [PATCH 064/239] remove use of hyperopt_conf.py --- docs/bot-usage.md | 5 ++- freqtrade/optimize/__init__.py | 10 ++---- freqtrade/optimize/hyperopt.py | 6 +--- freqtrade/tests/optimize/test_hyperopt.py | 7 ---- freqtrade/tests/optimize/test_optimize.py | 2 -- user_data/hyperopt_conf.py | 42 ----------------------- 6 files changed, 6 insertions(+), 66 deletions(-) delete mode 100644 user_data/hyperopt_conf.py diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 8079d9816..0ca99ec0a 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -160,9 +160,8 @@ the parameter `-l` or `--live`. ## Hyperopt commands -It is possible to use hyperopt for trading strategy optimization. -Hyperopt uses an internal json config return by `hyperopt_optimize_conf()` -located in `freqtrade/optimize/hyperopt_conf.py`. +To optimize your strategy, you can use hyperopt parameter hyperoptimization +to find optimal parameter values for your stategy. ``` usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index fc5d53114..867e8c7dc 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -11,8 +11,6 @@ from freqtrade import misc, constants from freqtrade.exchange import get_ticker_history from freqtrade.arguments import TimeRange -from user_data.hyperopt_conf import hyperopt_optimize_conf - logger = logging.getLogger(__name__) @@ -83,7 +81,7 @@ def load_tickerdata_file( def load_data(datadir: str, ticker_interval: str, - pairs: Optional[List[str]] = None, + pairs: List[str], refresh_pairs: Optional[bool] = False, timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]: """ @@ -92,14 +90,12 @@ def load_data(datadir: str, """ result = {} - _pairs = pairs or hyperopt_optimize_conf()['exchange']['pair_whitelist'] - # If the user force the refresh of pairs if refresh_pairs: logger.info('Download data for all pairs and store them in %s', datadir) - download_pairs(datadir, _pairs, ticker_interval, timerange=timerange) + download_pairs(datadir, pairs, ticker_interval, timerange=timerange) - for pair in _pairs: + for pair in pairs: pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) if pairdata: result[pair] = pairdata diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 878acc2dc..5acd05766 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -27,7 +27,6 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from user_data.hyperopt_conf import hyperopt_optimize_conf logger = logging.getLogger(__name__) @@ -596,11 +595,8 @@ def start(args: Namespace) -> None: # Monkey patch the configuration with hyperopt_conf.py configuration = Configuration(args) logger.info('Starting freqtrade in Hyperopt mode') + config = configuration.load_config() - optimize_config = hyperopt_optimize_conf() - config = configuration._load_common_config(optimize_config) - config = configuration._load_backtesting_config(config) - config = configuration._load_hyperopt_config(config) config['exchange']['key'] = '' config['exchange']['secret'] = '' diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 3edfe4393..4ef5762e1 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -23,8 +23,6 @@ def init_hyperopt(default_conf, mocker): global _HYPEROPT_INITIALIZED, _HYPEROPT if not _HYPEROPT_INITIALIZED: mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', - MagicMock(return_value=default_conf)) mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) _HYPEROPT = Hyperopt(default_conf) _HYPEROPT_INITIALIZED = True @@ -64,8 +62,6 @@ def test_start(mocker, default_conf, caplog) -> None: """ start_mock = MagicMock() mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', - MagicMock(return_value=default_conf)) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) args = [ @@ -182,7 +178,6 @@ def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) @@ -227,7 +222,6 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> conf.update({'epochs': 1}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) @@ -270,7 +264,6 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 3f358cfb8..bac8a6b36 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -326,8 +326,6 @@ def test_load_tickerdata_file() -> None: def test_init(default_conf, mocker) -> None: - conf = {'exchange': {'pair_whitelist': []}} - mocker.patch('freqtrade.optimize.hyperopt_optimize_conf', return_value=conf) assert {} == optimize.load_data( '', pairs=[], diff --git a/user_data/hyperopt_conf.py b/user_data/hyperopt_conf.py deleted file mode 100644 index c3a6e2a29..000000000 --- a/user_data/hyperopt_conf.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -File that contains the configuration for Hyperopt -""" - - -def hyperopt_optimize_conf() -> dict: - """ - This function is used to define which parameters Hyperopt must used. - The "pair_whitelist" is only used is your are using Hyperopt with MongoDB, - without MongoDB, Hyperopt will use the pair your have set in your config file. - :return: - """ - return { - 'max_open_trades': 3, - 'stake_currency': 'BTC', - 'stake_amount': 0.01, - "minimal_roi": { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, - }, - 'stoploss': -0.10, - "bid_strategy": { - "ask_last_balance": 0.0 - }, - "exchange": { - "name": "bittrex", - "pair_whitelist": [ - "ETH/BTC", - "LTC/BTC", - "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", - "XLM/BTC", - "NXT/BTC", - "POWR/BTC", - "ADA/BTC", - "XMR/BTC" - ] - } - } From 0c85febe76bb61cad1f7e376d7188fd634ac4f29 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 15 Jun 2018 10:59:09 +0300 Subject: [PATCH 065/239] remove all mongodb related code --- docs/bot-usage.md | 7 ++-- docs/hyperopt.md | 36 ------------------- docs/installation.md | 28 ++------------- freqtrade/arguments.py | 6 ---- freqtrade/configuration.py | 5 --- freqtrade/optimize/hyperopt.py | 36 ++++++------------- freqtrade/tests/optimize/test_hyperopt.py | 31 ---------------- .../tests/optimize/test_hyperopt_config.py | 16 --------- freqtrade/tests/test_configuration.py | 5 --- scripts/start-hyperopt-worker.py | 27 -------------- scripts/start-mongodb.py | 21 ----------- 11 files changed, 16 insertions(+), 202 deletions(-) delete mode 100644 freqtrade/tests/optimize/test_hyperopt_config.py delete mode 100755 scripts/start-hyperopt-worker.py delete mode 100755 scripts/start-mongodb.py diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0ca99ec0a..25fc78f0a 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -165,7 +165,7 @@ to find optimal parameter values for your stategy. ``` usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] - [--timerange TIMERANGE] [-e INT] [--use-mongodb] + [--timerange TIMERANGE] [-e INT] [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] optional arguments: @@ -175,11 +175,8 @@ optional arguments: --realistic-simulation uses max_open_trades from config to simulate real world limitations - --timerange TIMERANGE - specify what timerange of data to use. + --timerange TIMERANGE specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) - --use-mongodb parallelize evaluations with mongodb (requires mongod - in PATH) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate list. Default: all diff --git a/docs/hyperopt.md b/docs/hyperopt.md index a079e34df..2ad94896a 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -9,7 +9,6 @@ parameters with Hyperopt. - [Advanced Hyperopt notions](#advanced-notions) - [Understand the Guards and Triggers](#understand-the-guards-and-triggers) - [Execute Hyperopt](#execute-hyperopt) - - [Hyperopt with MongoDB](#hyperopt-with-mongoDB) - [Understand the hyperopts result](#understand-the-backtesting-result) ## Prepare Hyperopt @@ -194,41 +193,6 @@ Legal values are: - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` -### Hyperopt with MongoDB -Hyperopt with MongoDB, is like Hyperopt under steroids. As you saw by -executing the previous command is the execution takes a long time. -To accelerate it you can use hyperopt with MongoDB. - -To run hyperopt with MongoDb you will need 3 terminals. - -**Terminal 1: Start MongoDB** -```bash -cd -source .env/bin/activate -python3 scripts/start-mongodb.py -``` - -**Terminal 2: Start Hyperopt worker** -```bash -cd -source .env/bin/activate -python3 scripts/start-hyperopt-worker.py -``` - -**Terminal 3: Start Hyperopt with MongoDB** -```bash -cd -source .env/bin/activate -python3 ./freqtrade/main.py -c config.json hyperopt --use-mongodb -``` - -**Re-run an Hyperopt** -To re-run Hyperopt you have to delete the existing MongoDB table. -```bash -cd -rm -rf .hyperopt/mongodb/ -``` - ## Understand the hyperopts result Once Hyperopt is completed you can use the result to adding new buy signal. Given following result from hyperopt: diff --git a/docs/installation.md b/docs/installation.md index 9818529f6..b6688885b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -225,17 +225,7 @@ cd .. rm -rf ./ta-lib* ``` -#### 3. [Optional] Install MongoDB - -Install MongoDB if you plan to optimize your strategy with Hyperopt. - -```bash -sudo apt-get install mongodb-org -``` - -> Complete tutorial from Digital Ocean: [How to Install MongoDB on Ubuntu 16.04](https://www.digitalocean.com/community/tutorials/how-to-install-mongodb-on-ubuntu-16-04). - -#### 4. Install FreqTrade +#### 3. Install FreqTrade Clone the git repository: @@ -243,7 +233,7 @@ Clone the git repository: git clone https://github.com/freqtrade/freqtrade.git ``` -#### 5. Configure `freqtrade` as a `systemd` service +#### 4. Configure `freqtrade` as a `systemd` service From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. @@ -267,19 +257,7 @@ sudo loginctl enable-linger "$USER" brew install python3 git wget ta-lib ``` -#### 2. [Optional] Install MongoDB - -Install MongoDB if you plan to optimize your strategy with Hyperopt. - -```bash -curl -O https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.4.10.tgz -tar -zxvf mongodb-osx-ssl-x86_64-3.4.10.tgz -mkdir -p /env/mongodb -cp -R -n mongodb-osx-x86_64-3.4.10/ /env/mongodb -export PATH=/env/mongodb/bin:$PATH -``` - -#### 3. Install FreqTrade +#### 2. Install FreqTrade Clone the git repository: diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 331bb73a0..b392fb53e 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -203,12 +203,6 @@ class Arguments(object): type=int, metavar='INT', ) - parser.add_argument( - '--use-mongodb', - help='parallelize evaluations with mongodb (requires mongod in PATH)', - dest='mongodb', - action='store_true', - ) parser.add_argument( '-s', '--spaces', help='Specify which parameters to hyperopt. Space separate list. \ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 1f14df560..7c3a5eb4b 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -188,11 +188,6 @@ class Configuration(object): logger.info('Parameter --epochs detected ...') logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) - # If --mongodb is used we add it to the configuration - if 'mongodb' in self.args and self.args.mongodb: - config.update({'mongodb': self.args.mongodb}) - logger.info('Parameter --use-mongodb detected ...') - # If --spaces is used we add it to the configuration if 'spaces' in self.args and self.args.spaces: config.update({'spaces': self.args.spaces}) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5acd05766..978d1fe2c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -19,7 +19,6 @@ from typing import Dict, Any, Callable, Optional import numpy import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe -from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib @@ -507,32 +506,20 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - if self.config.get('mongodb'): - logger.info('Using mongodb ...') + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries logger.info( - 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' + 'Continuing with trials. Current: %d, Total: %d', + self.current_tries, + self.total_tries ) - db_name = 'freqtrade_hyperopt' - self.trials = MongoTrials( - arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), - exp_key='exp1' - ) - else: - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, self.signal_handler) - # read trials file if we have one - if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: - self.trials = self.read_trials() - - self.current_tries = len(self.trials.results) - self.total_tries += self.current_tries - logger.info( - 'Continuing with trials. Current: %d, Total: %d', - self.current_tries, - self.total_tries - ) - try: best_parameters = fmin( fn=self.generate_optimizer, @@ -588,7 +575,6 @@ def start(args: Namespace) -> None: """ # Remove noisy log messages - logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) # Initialize configuration diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 4ef5762e1..ce7cd77c8 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -247,7 +247,6 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'epochs': 1}) - conf.update({'mongodb': False}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) @@ -341,7 +340,6 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'epochs': 1}) - conf.update({'mongodb': False}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) @@ -353,35 +351,6 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: mock_fmin.assert_called_once() -def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None: - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - mock_mongotrials = mocker.patch( - 'freqtrade.optimize.hyperopt.MongoTrials', - return_value=create_trials(mocker) - ) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'mongodb': True}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - - hyperopt = Hyperopt(conf) - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - mock_mongotrials.assert_called_once() - mock_fmin.assert_called_once() - - -# test log_trials_result -# test buy_strategy_generator def populate_buy_trend -# test optimizer if 'ro_t1' in params - def test_format_results(init_hyperopt): """ Test Hyperopt.format_results() diff --git a/freqtrade/tests/optimize/test_hyperopt_config.py b/freqtrade/tests/optimize/test_hyperopt_config.py deleted file mode 100644 index aa9424826..000000000 --- a/freqtrade/tests/optimize/test_hyperopt_config.py +++ /dev/null @@ -1,16 +0,0 @@ -# pragma pylint: disable=missing-docstring,W0212 - -from user_data.hyperopt_conf import hyperopt_optimize_conf - - -def test_hyperopt_optimize_conf(): - hyperopt_conf = hyperopt_optimize_conf() - - assert "max_open_trades" in hyperopt_conf - assert "stake_currency" in hyperopt_conf - assert "stake_amount" in hyperopt_conf - assert "minimal_roi" in hyperopt_conf - assert "stoploss" in hyperopt_conf - assert "bid_strategy" in hyperopt_conf - assert "exchange" in hyperopt_conf - assert "pair_whitelist" in hyperopt_conf['exchange'] diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index caaddbf25..212df2e96 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -310,7 +310,6 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: arglist = [ 'hyperopt', '--epochs', '10', - '--use-mongodb', '--spaces', 'all', ] @@ -324,10 +323,6 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert log_has('Parameter --epochs detected ...', caplog.record_tuples) assert log_has('Will run Hyperopt with for 10 epochs ...', caplog.record_tuples) - assert 'mongodb' in config - assert config['mongodb'] is True - assert log_has('Parameter --use-mongodb detected ...', caplog.record_tuples) - assert 'spaces' in config assert config['spaces'] == ['all'] assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py deleted file mode 100755 index 8b0ae6326..000000000 --- a/scripts/start-hyperopt-worker.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -import multiprocessing -import os -import subprocess - -PROC_COUNT = multiprocessing.cpu_count() - 1 -DB_NAME = 'freqtrade_hyperopt' -WORK_DIR = os.path.join( - os.path.sep, - os.path.abspath(os.path.dirname(__file__)), - '..', '.hyperopt', 'worker' -) -if not os.path.exists(WORK_DIR): - os.makedirs(WORK_DIR) - -# Spawn workers -command = [ - 'hyperopt-mongo-worker', - '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), - '--poll-interval=0.1', - '--workdir={}'.format(WORK_DIR), -] -processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] - -# Join all workers -for proc in processes: - proc.wait() diff --git a/scripts/start-mongodb.py b/scripts/start-mongodb.py deleted file mode 100755 index 910ee9233..000000000 --- a/scripts/start-mongodb.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 - -import os -import subprocess - - -DB_PATH = os.path.join( - os.path.sep, - os.path.abspath(os.path.dirname(__file__)), - '..', '.hyperopt', 'mongodb' -) -if not os.path.exists(DB_PATH): - os.makedirs(DB_PATH) - -subprocess.Popen([ - 'mongod', - '--bind_ip=127.0.0.1', - '--port=1234', - '--nohttpinterface', - '--dbpath={}'.format(DB_PATH), -]).wait() From 7e2e7946c5b12abffd222420d74156208ef41e59 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 16 Jun 2018 09:06:14 +0300 Subject: [PATCH 066/239] also unit tests now need config.json --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 88121945f..3f041f5dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,16 +16,15 @@ install: - pip install --upgrade flake8 coveralls pytest-random-order mypy - pip install -r requirements.txt - pip install -e . +- cp config.json.example config.json jobs: include: - script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls - script: - - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting - script: - - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 - script: flake8 freqtrade - script: mypy freqtrade From fa00157d12f7faad9cd778d4d825019aa88e5ba4 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 16 Jun 2018 13:42:25 +0200 Subject: [PATCH 067/239] Fix fiat_convert missing mockups --- freqtrade/tests/test_fiat_convert.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 24f0f776b..2fb9219ca 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -40,7 +40,8 @@ def test_pair_convertion_object(): assert pair_convertion.price == 30000.123 -def test_fiat_convert_is_supported(): +def test_fiat_convert_is_supported(mocker): + patch_coinmarketcap(mocker) fiat_convert = CryptoToFiatConverter() assert fiat_convert._is_supported_fiat(fiat='USD') is True assert fiat_convert._is_supported_fiat(fiat='usd') is True @@ -48,7 +49,9 @@ def test_fiat_convert_is_supported(): assert fiat_convert._is_supported_fiat(fiat='ABC') is False -def test_fiat_convert_add_pair(): +def test_fiat_convert_add_pair(mocker): + patch_coinmarketcap(mocker) + fiat_convert = CryptoToFiatConverter() pair_len = len(fiat_convert._pairs) @@ -70,11 +73,8 @@ def test_fiat_convert_add_pair(): def test_fiat_convert_find_price(mocker): - api_mock = MagicMock(return_value={ - 'price_usd': 12345.0, - 'price_eur': 13000.2 - }) - mocker.patch('freqtrade.fiat_convert.Market.ticker', api_mock) + patch_coinmarketcap(mocker) + fiat_convert = CryptoToFiatConverter() with pytest.raises(ValueError, match=r'The fiat ABC is not supported.'): @@ -92,17 +92,15 @@ def test_fiat_convert_find_price(mocker): def test_fiat_convert_unsupported_crypto(mocker, caplog): mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._cryptomap', return_value=[]) + patch_coinmarketcap(mocker) fiat_convert = CryptoToFiatConverter() assert fiat_convert._find_price(crypto_symbol='CRYPTO_123', fiat_symbol='EUR') == 0.0 assert log_has('unsupported crypto-symbol CRYPTO_123 - returning 0.0', caplog.record_tuples) def test_fiat_convert_get_price(mocker): - api_mock = MagicMock(return_value={ - 'price_usd': 28000.0, - 'price_eur': 15000.0 - }) - mocker.patch('freqtrade.fiat_convert.Market.ticker', api_mock) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=28000.0) fiat_convert = CryptoToFiatConverter() @@ -172,8 +170,9 @@ def test_fiat_init_network_exception(mocker): assert length_cryptomap == 0 -def test_fiat_convert_without_network(): +def test_fiat_convert_without_network(mocker): # Because CryptoToFiatConverter is a Singleton we reset the value of _coinmarketcap + patch_coinmarketcap(mocker) fiat_convert = CryptoToFiatConverter() @@ -186,6 +185,7 @@ def test_fiat_convert_without_network(): def test_convert_amount(mocker): + patch_coinmarketcap(mocker) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter.get_price', return_value=12345.0) fiat_convert = CryptoToFiatConverter() From 7564f7e526cfa8a1f8bf04155d26e3e9e9eadade Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 16 Jun 2018 13:49:03 +0200 Subject: [PATCH 068/239] fix hyperopt test when no config.json exists --- freqtrade/tests/optimize/test_hyperopt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f8dce5fd6..9b66a29e2 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -61,6 +61,10 @@ def test_start(mocker, default_conf, caplog) -> None: Test start() function """ start_mock = MagicMock() + mocker.patch( + 'freqtrade.configuration.Configuration._load_config_file', + lambda *args, **kwargs: default_conf + ) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) From 17801871b15ea76710528c8c30499bc9e8b8a314 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 16 Jun 2018 14:23:06 +0200 Subject: [PATCH 069/239] Update ccxt from 1.14.198 to 1.14.201 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e8240fd97..7e01b10aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.198 +ccxt==1.14.201 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 972736f0abd34945b4505a351aece6e4436c82a2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 16 Jun 2018 20:55:35 +0200 Subject: [PATCH 070/239] increase test-coverate for configureation --- freqtrade/tests/test_configuration.py | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 212df2e96..019c0c09d 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -13,6 +13,7 @@ from jsonschema import ValidationError from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration +from freqtrade.constants import DEFAULT_DB_PROD_URL, DEFAULT_DB_DRYRUN_URL from freqtrade.tests.conftest import log_has from freqtrade import OperationalException @@ -140,6 +141,43 @@ def test_load_config_with_params(default_conf, mocker) -> None: assert validated_conf.get('strategy_path') == '/some/path' assert validated_conf.get('db_url') == 'sqlite:///someurl' + conf = default_conf.copy() + conf["dry_run"] = False + del conf["db_url"] + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(conf) + )) + + arglist = [ + '--dynamic-whitelist', '10', + '--strategy', 'TestStrategy', + '--strategy-path', '/some/path' + ] + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration.load_config() + assert validated_conf.get('db_url') == DEFAULT_DB_PROD_URL + + # Test dry=run with ProdURL + conf = default_conf.copy() + conf["dry_run"] = True + conf["db_url"] = DEFAULT_DB_PROD_URL + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(conf) + )) + + arglist = [ + '--dynamic-whitelist', '10', + '--strategy', 'TestStrategy', + '--strategy-path', '/some/path' + ] + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration.load_config() + assert validated_conf.get('db_url') == DEFAULT_DB_DRYRUN_URL + def test_load_custom_strategy(default_conf, mocker) -> None: """ From 7cfd99d17f574de139ccd11e5cdd3b2a5b4ca124 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 16 Jun 2018 21:00:45 +0200 Subject: [PATCH 071/239] exclude __main__.py from coveralls - if __name__ == '__main__' is close to untestable - and should do nothing other than calling another function. --- .coveragerc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 95eea4f8f..80ed2af23 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,4 +2,5 @@ omit = scripts/* freqtrade/tests/* - freqtrade/vendor/* \ No newline at end of file + freqtrade/vendor/* + freatrade/__main__.py From 90a7fb603d50e52aced0cfaf1ed6221c469a8e10 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 16 Jun 2018 21:28:41 +0200 Subject: [PATCH 072/239] fix typo in coverage-omit --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 80ed2af23..4bd5b63fa 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,4 +3,4 @@ omit = scripts/* freqtrade/tests/* freqtrade/vendor/* - freatrade/__main__.py + freqtrade/__main__.py From eb909068c5c5d37b8a574b722529f47aab276ffc Mon Sep 17 00:00:00 2001 From: Anton Date: Sun, 17 Jun 2018 02:23:12 +0300 Subject: [PATCH 073/239] Add minimal pair stake amount check --- freqtrade/analyze.py | 7 + freqtrade/freqtradebot.py | 46 ++++- freqtrade/tests/conftest.py | 90 +++++++- freqtrade/tests/rpc/test_rpc.py | 36 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 46 +++-- freqtrade/tests/test_freqtradebot.py | 251 ++++++++++++++++++----- 6 files changed, 388 insertions(+), 88 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index f18ae291c..2ef3a7515 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -98,6 +98,13 @@ class Analyze(object): """ return self.strategy.ticker_interval + def get_stoploss(self) -> float: + """ + Return stoploss to use + :return: Strategy stoploss value to use + """ + return self.strategy.stoploss + def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1477a931e..a5612371d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -254,13 +254,7 @@ class FreqtradeBot(object): if open_trades >= self.config['max_open_trades']: logger.warning('Can\'t open a new trade: max number of trades is reached') return None - trade_stake_amount = avaliable_amount / (self.config['max_open_trades'] - open_trades) - if trade_stake_amount < 0.0005: - raise DependencyException( - 'Available balance(%f %s) is lower than minimal' % ( - avaliable_amount, self.config['stake_currency']) - ) - return trade_stake_amount + return avaliable_amount / (self.config['max_open_trades'] - open_trades) # Check if stake_amount is fulfilled if avaliable_amount < stake_amount: @@ -272,6 +266,34 @@ class FreqtradeBot(object): return stake_amount + def _get_min_pair_stake_amount(self, pair: str, price: float) -> Optional[float]: + markets = exchange.get_markets() + markets = [m for m in markets if m['symbol'] == pair] + if not markets: + raise ValueError(f'Can\'t get market information for symbol {pair}') + + market = markets[0] + + if 'limits' not in market: + return None + + min_stake_amounts = [] + if 'cost' in market['limits'] and 'min' in market['limits']['cost']: + min_stake_amounts.append(market['limits']['cost']['min']) + + if 'amount' in market['limits'] and 'min' in market['limits']['amount']: + min_stake_amounts.append(market['limits']['amount']['min'] * price) + + if not min_stake_amounts: + return None + + amount_reserve_percent = 1 - 0.05 # reserve 5% + stoploss + if self.analyze.get_stoploss() is not None: + amount_reserve_percent += self.analyze.get_stoploss() + # it should not be more than 50% + amount_reserve_percent = max(amount_reserve_percent, 0.5) + return min(min_stake_amounts)/amount_reserve_percent + def create_trade(self) -> bool: """ Checks the implemented trading indicator(s) for a randomly picked pair, @@ -314,8 +336,16 @@ class FreqtradeBot(object): pair_url = exchange.get_pair_detail_url(pair) # Calculate amount buy_limit = self.get_target_bid(exchange.get_ticker(pair)) - amount = stake_amount / buy_limit + min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) + if min_stake_amount is not None and min_stake_amount > stake_amount: + logger.warning( + f'Can\'t open a new trade for {pair_s}: stake amount' + f' is too small ({stake_amount} < {min_stake_amount})' + ) + return False + + amount = stake_amount / buy_limit order_id = exchange.buy(pair, buy_limit, amount)['id'] stake_amount_fiat = self.fiat_converter.convert_amount( diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 1311687b7..5eed0a08f 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -174,7 +174,10 @@ def markets(): 'max': 1000, }, 'price': 500000, - 'cost': 500000, + 'cost': { + 'min': 1, + 'max': 500000, + }, }, 'info': '', }, @@ -196,7 +199,10 @@ def markets(): 'max': 1000, }, 'price': 500000, - 'cost': 500000, + 'cost': { + 'min': 1, + 'max': 500000, + }, }, 'info': '', }, @@ -218,7 +224,85 @@ def markets(): 'max': 1000, }, 'price': 500000, - 'cost': 500000, + 'cost': { + 'min': 1, + 'max': 500000, + }, + }, + 'info': '', + }, + { + 'id': 'ltcbtc', + 'symbol': 'LTC/BTC', + 'base': 'LTC', + 'quote': 'BTC', + 'active': False, + 'precision': { + 'price': 8, + 'amount': 8, + 'cost': 8, + }, + 'lot': 0.00000001, + 'limits': { + 'amount': { + 'min': 0.01, + 'max': 1000, + }, + 'price': 500000, + 'cost': { + 'min': 1, + 'max': 500000, + }, + }, + 'info': '', + }, + { + 'id': 'xrpbtc', + 'symbol': 'XRP/BTC', + 'base': 'XRP', + 'quote': 'BTC', + 'active': False, + 'precision': { + 'price': 8, + 'amount': 8, + 'cost': 8, + }, + 'lot': 0.00000001, + 'limits': { + 'amount': { + 'min': 0.01, + 'max': 1000, + }, + 'price': 500000, + 'cost': { + 'min': 1, + 'max': 500000, + }, + }, + 'info': '', + }, + { + 'id': 'neobtc', + 'symbol': 'NEO/BTC', + 'base': 'NEO', + 'quote': 'BTC', + 'active': False, + 'precision': { + 'price': 8, + 'amount': 8, + 'cost': 8, + }, + 'lot': 0.00000001, + 'limits': { + 'amount': { + 'min': 0.01, + 'max': 1000, + }, + 'price': 500000, + 'cost': { + 'min': 1, + 'max': 500000, + }, }, 'info': '', } diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 5cdd22c7a..f947cc9f0 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -23,7 +23,7 @@ def prec_satoshi(a, b) -> float: # Unit tests -def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: +def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: """ Test rpc_trade_status() method """ @@ -34,7 +34,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -72,7 +73,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: assert trade.find('[ETH/BTC]') >= 0 -def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: +def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: """ Test rpc_status_table() method """ @@ -83,7 +84,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -107,7 +109,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: def test_rpc_daily_profit(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: + limit_buy_order, limit_sell_order, markets, mocker) -> None: """ Test rpc_daily_profit() method """ @@ -118,7 +120,8 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -160,7 +163,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, mocker) -> None: + limit_buy_order, limit_sell_order, markets, mocker) -> None: """ Test rpc_trade_statistics() method """ @@ -175,7 +178,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -237,7 +241,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) -def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, +def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, ticker_sell_up, limit_buy_order, limit_sell_order): """ Test rpc_trade_statistics() method @@ -253,7 +257,8 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -400,7 +405,7 @@ def test_rpc_stop(mocker, default_conf) -> None: assert freqtradebot.state == State.STOPPED -def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: +def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: """ Test rpc_forcesell() method """ @@ -422,6 +427,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: } ), get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -519,7 +525,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: def test_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: + limit_sell_order, markets, mocker) -> None: """ Test rpc_performance() method """ @@ -531,7 +537,8 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, validate_pairs=MagicMock(), get_balances=MagicMock(return_value=ticker), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -558,7 +565,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, assert prec_satoshi(res[0]['profit'], 6.2) -def test_rpc_count(mocker, default_conf, ticker, fee) -> None: +def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: """ Test rpc_count() method """ @@ -571,6 +578,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 0919455ad..d10abd8b5 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -233,7 +233,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: ) -def test_status(default_conf, update, mocker, fee, ticker) -> None: +def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: """ Test _status() method """ @@ -250,6 +250,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: get_ticker=ticker, get_pair_detail_url=MagicMock(), get_fee=fee, + get_markets=markets ) msg_mock = MagicMock() status_table = MagicMock() @@ -278,7 +279,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: assert status_table.call_count == 1 -def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: +def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> None: """ Test _status() method """ @@ -289,6 +290,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, + get_markets=markets ) msg_mock = MagicMock() status_table = MagicMock() @@ -324,7 +326,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: assert '[ETH/BTC]' in msg_mock.call_args_list[0][0][0] -def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: +def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) -> None: """ Test _status_table() method """ @@ -336,6 +338,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, + get_markets=markets ) msg_mock = MagicMock() mocker.patch.multiple( @@ -377,7 +380,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: + limit_sell_order, markets, mocker) -> None: """ Test _daily() method """ @@ -391,7 +394,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) msg_mock = MagicMock() mocker.patch.multiple( @@ -489,7 +493,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, mocker) -> None: + limit_buy_order, limit_sell_order, markets, mocker) -> None: """ Test _profit() method """ @@ -500,7 +504,8 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) msg_mock = MagicMock() mocker.patch.multiple( @@ -768,7 +773,8 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: assert 'Reloading config' in msg_mock.call_args_list[0][0][0] -def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, mocker) -> None: +def test_forcesell_handle(default_conf, update, ticker, fee, + ticker_sell_up, markets, mocker) -> None: """ Test _forcesell() method """ @@ -781,7 +787,8 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, moc 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -808,7 +815,8 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, moc assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] -def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_down, mocker) -> None: +def test_forcesell_down_handle(default_conf, update, ticker, fee, + ticker_sell_down, markets, mocker) -> None: """ Test _forcesell() method """ @@ -821,7 +829,8 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_do 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -852,7 +861,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_do assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] -def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None: +def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: """ Test _forcesell() method """ @@ -866,7 +875,8 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtradebot = FreqtradeBot(default_conf) @@ -930,7 +940,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: def test_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: + limit_buy_order, limit_sell_order, markets, mocker) -> None: """ Test _performance() method """ @@ -946,7 +956,8 @@ def test_performance_handle(default_conf, update, ticker, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) @@ -994,7 +1005,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: assert 'not running' in msg_mock.call_args_list[0][0][0] -def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: +def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> None: """ Test _count() method """ @@ -1010,7 +1021,8 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - buy=MagicMock(return_value={'id': 'mocked_order_id'}) + buy=MagicMock(return_value={'id': 'mocked_order_id'}), + get_markets=markets ) mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) freqtradebot = FreqtradeBot(default_conf) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 80b66735d..badc1fc3d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -255,20 +255,12 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade._get_trade_stake_amount() - # test UNLIMITED_STAKE_AMOUNT - conf = deepcopy(default_conf) - conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT - conf['max_open_trades'] = 2 - freqtrade = FreqtradeBot(conf) - - with pytest.raises(DependencyException, match=r'.*is lower than minimal.*'): - freqtrade._get_trade_stake_amount() - def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, limit_buy_order, fee, + markets, mocker) -> None: """ Test get_trade_stake_amount() method @@ -283,6 +275,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -314,28 +307,119 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result is None -def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_get_min_pair_stake_amount(mocker, default_conf) -> None: """ - Test create_trade() method + Test get_trade_stake_amount() method """ - patch_get_signal(mocker) + patch_RPCManager(mocker) - patch_coinmarketcap(mocker) - mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), - get_fee=fee, - ) + mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.freqtradebot.Analyze.get_stoploss', MagicMock(return_value=-0.05)) freqtrade = FreqtradeBot(default_conf) - with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade.create_trade() + # no pair found + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC' + }]) + ) + with pytest.raises(ValueError, match=r'.*get market information.*'): + freqtrade._get_min_pair_stake_amount('BNB/BTC', 1) + + # no 'limits' section + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC' + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # empty 'limits' section + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': {} + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # empty 'cost'/'amount' section + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {}, + 'amount': {} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # min cost is set + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {'min': 2}, + 'amount': {} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result == 2 / 0.9 + + # min amount is set + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {}, + 'amount': {'min': 2} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 2) + assert result == 2 * 2 / 0.9 + + # min amount and cost are set (cost is minimal) + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {'min': 2}, + 'amount': {'min': 2} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 2) + assert result == min(2, 2 * 2) / 0.9 + + # min amount and cost are set (amount is minial) + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {'min': 8}, + 'amount': {'min': 2} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 2) + assert result == min(8, 2 * 2) / 0.9 -def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: """ Test create_trade() method """ @@ -348,6 +432,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) # Save state of current whitelist @@ -371,7 +456,31 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non assert whitelist == default_conf['exchange']['pair_whitelist'] -def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + + with pytest.raises(DependencyException, match=r'.*stake amount.*'): + freqtrade.create_trade() + + +def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: """ Test create_trade() method """ @@ -385,6 +494,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, get_ticker=ticker, buy=buy_mock, get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -396,7 +506,34 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, assert rate * amount >= conf['stake_amount'] -def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=buy_mock, + get_fee=fee, + get_markets=markets + ) + + conf = deepcopy(default_conf) + conf['stake_amount'] = 0.000000005 + freqtrade = FreqtradeBot(conf) + + result = freqtrade.create_trade() + assert result is False + + +def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: """ Test create_trade() method """ @@ -410,6 +547,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, fee, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) conf['max_open_trades'] = 0 @@ -421,7 +559,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, fee, assert freqtrade._get_trade_stake_amount() is None -def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: """ Test create_trade() method """ @@ -434,6 +572,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocke get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -448,7 +587,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocke def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, - limit_buy_order, fee, mocker) -> None: + limit_buy_order, fee, markets, mocker) -> None: """ Test create_trade() method """ @@ -461,6 +600,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -736,7 +876,8 @@ def test_process_maybe_execute_sell_exception(mocker, default_conf, assert log_has('Unable to sell trade: ', caplog.record_tuples) -def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mocker) -> None: +def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, + fee, markets, mocker) -> None: """ Test check_handle() method """ @@ -752,7 +893,8 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), - get_fee=fee + get_fee=fee, + get_markets=markets ) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) @@ -780,7 +922,8 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock assert trade.close_date is not None -def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: """ Test check_handle() method """ @@ -797,6 +940,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) freqtrade = FreqtradeBot(conf) @@ -838,7 +982,8 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, assert freqtrade.handle_trade(trades[0]) is True -def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, caplog) -> None: +def test_handle_trade_roi(default_conf, ticker, limit_buy_order, + fee, mocker, markets, caplog) -> None: """ Test check_handle() method """ @@ -855,6 +1000,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, ca get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) @@ -875,7 +1021,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, ca def test_handle_trade_experimental( - default_conf, ticker, limit_buy_order, fee, mocker, caplog) -> None: + default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None: """ Test check_handle() method """ @@ -892,6 +1038,7 @@ def test_handle_trade_experimental( get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) @@ -909,7 +1056,8 @@ def test_handle_trade_experimental( assert log_has('Sell signal received. Selling..', caplog.record_tuples) -def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fee, mocker) -> None: +def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, + fee, markets, mocker) -> None: """ Test check_handle() method """ @@ -922,6 +1070,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fe get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) freqtrade = FreqtradeBot(default_conf) @@ -1160,7 +1309,7 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: assert cancel_order_mock.call_count == 1 -def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: +def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: """ Test execute_sell() method with a ticker going UP """ @@ -1171,7 +1320,8 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) freqtrade = FreqtradeBot(default_conf) @@ -1201,7 +1351,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] -def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) -> None: +def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: """ Test execute_sell() method with a ticker going DOWN """ @@ -1213,7 +1363,8 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtrade = FreqtradeBot(default_conf) @@ -1242,7 +1393,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, - ticker_sell_up, mocker) -> None: + ticker_sell_up, markets, mocker) -> None: """ Test execute_sell() method with a ticker going DOWN and with a bot config empty """ @@ -1253,7 +1404,8 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtrade = FreqtradeBot(default_conf) @@ -1283,7 +1435,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, - ticker_sell_down, mocker) -> None: + ticker_sell_down, markets, mocker) -> None: """ Test execute_sell() method with a ticker going DOWN and with a bot config empty """ @@ -1294,7 +1446,8 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), get_ticker=ticker, - get_fee=fee + get_fee=fee, + get_markets=markets ) freqtrade = FreqtradeBot(default_conf) @@ -1321,7 +1474,8 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] -def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, + fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled """ @@ -1339,6 +1493,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, mock }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) conf['experimental'] = { @@ -1354,7 +1509,8 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, mock assert freqtrade.handle_trade(trade) is True -def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, + fee, markets, mocker) -> None: """ Test sell_profit_only feature when disabled """ @@ -1372,6 +1528,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, moc }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) conf['experimental'] = { @@ -1387,7 +1544,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, moc assert freqtrade.handle_trade(trade) is True -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1405,6 +1562,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) conf['experimental'] = { @@ -1420,7 +1578,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker assert freqtrade.handle_trade(trade) is False -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1438,6 +1596,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) From ad0549414b7434fbd31841dbffe36dbd34dce1ac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 11:34:12 +0200 Subject: [PATCH 074/239] Revert "also unit tests now need config.json" This reverts commit 7e2e7946c5b12abffd222420d74156208ef41e59. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3f041f5dd..88121945f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,15 +16,16 @@ install: - pip install --upgrade flake8 coveralls pytest-random-order mypy - pip install -r requirements.txt - pip install -e . -- cp config.json.example config.json jobs: include: - script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls - script: + - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting - script: + - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 - script: flake8 freqtrade - script: mypy freqtrade From fef267a0dcf8f9256961866a48dba763f1652699 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 17 Jun 2018 14:23:05 +0200 Subject: [PATCH 075/239] Update ccxt from 1.14.201 to 1.14.202 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7e01b10aa..9d572304d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.201 +ccxt==1.14.202 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 21edcbdc2747e8ea7b31a8a61eec3420835725ec Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 12:41:33 +0200 Subject: [PATCH 076/239] Refactor exchange to class --- freqtrade/analyze.py | 6 +- freqtrade/exchange/__init__.py | 658 +++++++++++++++--------------- freqtrade/freqtradebot.py | 49 +-- freqtrade/optimize/__init__.py | 6 +- freqtrade/optimize/backtesting.py | 8 +- 5 files changed, 356 insertions(+), 371 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index f18ae291c..aa64f04e5 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -10,7 +10,7 @@ import arrow from pandas import DataFrame, to_datetime from freqtrade import constants -from freqtrade.exchange import get_ticker_history +from freqtrade.exchange import Exchange from freqtrade.persistence import Trade from freqtrade.strategy.resolver import StrategyResolver, IStrategy @@ -110,14 +110,14 @@ class Analyze(object): dataframe = self.populate_sell_trend(dataframe) return dataframe - def get_signal(self, pair: str, interval: str) -> Tuple[bool, bool]: + def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC :param interval: Interval to use (in min) :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ - ticker_hist = get_ticker_history(pair, interval) + ticker_hist = exchange.get_ticker_history(pair, interval) if not ticker_hist: logger.warning('Empty ticker history for pair %s', pair) return False, False diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 54d564f04..5ac5f8a0b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -12,16 +12,8 @@ from freqtrade import constants, OperationalException, DependencyException, Temp logger = logging.getLogger(__name__) -# Current selected exchange -_API: ccxt.Exchange = None - -_CONF: Dict = {} API_RETRY_COUNT = 4 -_CACHED_TICKER: Dict[str, Any] = {} - -# Holds all open sell orders for dry_run -_DRY_RUN_OPEN_ORDERS: Dict[str, Any] = {} # Urls to exchange markets, insert quote and base with .format() _EXCHANGE_URLS = { @@ -74,364 +66,354 @@ def init_ccxt(exchange_config: dict) -> ccxt.Exchange: return api -def init(config: dict) -> None: - """ - Initializes this module with the given config, - it does basic validation whether the specified - exchange and pairs are valid. - :param config: config to use - :return: None - """ - global _CONF, _API +class Exchange(object): - _CONF.update(config) + # Current selected exchange + _API: ccxt.Exchange = None + _CONF: Dict = {} + _CACHED_TICKER: Dict[str, Any] = {} - if config['dry_run']: - logger.info('Instance is running with dry_run enabled') + # Holds all open sell orders for dry_run + _DRY_RUN_OPEN_ORDERS: Dict[str, Any] = {} - exchange_config = config['exchange'] - _API = init_ccxt(exchange_config) + def __init__(self, config: dict) -> None: + """ + Initializes this module with the given config, + it does basic validation whether the specified + exchange and pairs are valid. + :param config: config to use + :return: None + """ + self._API - logger.info('Using Exchange "%s"', get_name()) + self._CONF.update(config) - # Check if all pairs are available - validate_pairs(config['exchange']['pair_whitelist']) + if config['dry_run']: + logger.info('Instance is running with dry_run enabled') + exchange_config = config['exchange'] + self._API = init_ccxt(exchange_config) -def validate_pairs(pairs: List[str]) -> None: - """ - Checks if all given pairs are tradable on the current exchange. - Raises OperationalException if one pair is not available. - :param pairs: list of pairs - :return: None - """ + logger.info('Using Exchange "%s"', self.get_name()) - try: - markets = _API.load_markets() - except ccxt.BaseError as e: - logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) - return + # Check if all pairs are available + self.validate_pairs(config['exchange']['pair_whitelist']) - stake_cur = _CONF['stake_currency'] - for pair in pairs: - # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs - # TODO: add a support for having coins in BTC/USDT format - if not pair.endswith(stake_cur): - raise OperationalException( - f'Pair {pair} not compatible with stake_currency: {stake_cur}') - if pair not in markets: - raise OperationalException( - f'Pair {pair} is not available at {get_name()}') + def get_name(self) -> str: + return self._API.name + def get_id(self) -> str: + return self._API.id -def exchange_has(endpoint: str) -> bool: - """ - Checks if exchange implements a specific API endpoint. - Wrapper around ccxt 'has' attribute - :param endpoint: Name of endpoint (e.g. 'fetchOHLCV', 'fetchTickers') - :return: bool - """ - return endpoint in _API.has and _API.has[endpoint] + def validate_pairs(self, pairs: List[str]) -> None: + """ + Checks if all given pairs are tradable on the current exchange. + Raises OperationalException if one pair is not available. + :param pairs: list of pairs + :return: None + """ - -def buy(pair: str, rate: float, amount: float) -> Dict: - if _CONF['dry_run']: - global _DRY_RUN_OPEN_ORDERS - order_id = f'dry_run_buy_{randint(0, 10**6)}' - _DRY_RUN_OPEN_ORDERS[order_id] = { - 'pair': pair, - 'price': rate, - 'amount': amount, - 'type': 'limit', - 'side': 'buy', - 'remaining': 0.0, - 'datetime': arrow.utcnow().isoformat(), - 'status': 'closed', - 'fee': None - } - return {'id': order_id} - - try: - return _API.create_limit_buy_order(pair, amount, rate) - except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create limit buy order on market {pair}.' - f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not create limit buy order on market {pair}.' - f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not place buy order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -def sell(pair: str, rate: float, amount: float) -> Dict: - if _CONF['dry_run']: - global _DRY_RUN_OPEN_ORDERS - order_id = f'dry_run_sell_{randint(0, 10**6)}' - _DRY_RUN_OPEN_ORDERS[order_id] = { - 'pair': pair, - 'price': rate, - 'amount': amount, - 'type': 'limit', - 'side': 'sell', - 'remaining': 0.0, - 'datetime': arrow.utcnow().isoformat(), - 'status': 'closed' - } - return {'id': order_id} - - try: - return _API.create_limit_sell_order(pair, amount, rate) - except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create limit sell order on market {pair}.' - f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not create limit sell order on market {pair}.' - f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -@retrier -def get_balance(currency: str) -> float: - if _CONF['dry_run']: - return 999.9 - - # ccxt exception is already handled by get_balances - balances = get_balances() - balance = balances.get(currency) - if balance is None: - raise TemporaryError( - f'Could not get {currency} balance due to malformed exchange response: {balances}') - return balance['free'] - - -@retrier -def get_balances() -> dict: - if _CONF['dry_run']: - return {} - - try: - balances = _API.fetch_balance() - # Remove additional info from ccxt results - balances.pop("info", None) - balances.pop("free", None) - balances.pop("total", None) - balances.pop("used", None) - - return balances - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get balance due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -@retrier -def get_tickers() -> Dict: - try: - return _API.fetch_tickers() - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {_API.name} does not support fetching tickers in batch.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -@retrier -def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: - global _CACHED_TICKER - if refresh or pair not in _CACHED_TICKER.keys(): try: - data = _API.fetch_ticker(pair) + markets = self._API.load_markets() + except ccxt.BaseError as e: + logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) + return + + stake_cur = self._CONF['stake_currency'] + for pair in pairs: + # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs + # TODO: add a support for having coins in BTC/USDT format + if not pair.endswith(stake_cur): + raise OperationalException( + f'Pair {pair} not compatible with stake_currency: {stake_cur}') + if pair not in markets: + raise OperationalException( + f'Pair {pair} is not available at {self.get_name()}') + + def exchange_has(self, endpoint: str) -> bool: + """ + Checks if exchange implements a specific API endpoint. + Wrapper around ccxt 'has' attribute + :param endpoint: Name of endpoint (e.g. 'fetchOHLCV', 'fetchTickers') + :return: bool + """ + return endpoint in self._API.has and self._API.has[endpoint] + + def buy(self, pair: str, rate: float, amount: float) -> Dict: + if self._CONF['dry_run']: + order_id = f'dry_run_buy_{randint(0, 10**6)}' + self._DRY_RUN_OPEN_ORDERS[order_id] = { + 'pair': pair, + 'price': rate, + 'amount': amount, + 'type': 'limit', + 'side': 'buy', + 'remaining': 0.0, + 'datetime': arrow.utcnow().isoformat(), + 'status': 'closed', + 'fee': None + } + return {'id': order_id} + + try: + return self._API.create_limit_buy_order(pair, amount, rate) + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create limit buy order on market {pair}.' + f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not create limit buy order on market {pair}.' + f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place buy order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + def sell(self, pair: str, rate: float, amount: float) -> Dict: + if self._CONF['dry_run']: + order_id = f'dry_run_sell_{randint(0, 10**6)}' + self._DRY_RUN_OPEN_ORDERS[order_id] = { + 'pair': pair, + 'price': rate, + 'amount': amount, + 'type': 'limit', + 'side': 'sell', + 'remaining': 0.0, + 'datetime': arrow.utcnow().isoformat(), + 'status': 'closed' + } + return {'id': order_id} + + try: + return self._API.create_limit_sell_order(pair, amount, rate) + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create limit sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not create limit sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_balance(self, currency: str) -> float: + if self._CONF['dry_run']: + return 999.9 + + # ccxt exception is already handled by get_balances + balances = self.get_balances() + balance = balances.get(currency) + if balance is None: + raise TemporaryError( + f'Could not get {currency} balance due to malformed exchange response: {balances}') + return balance['free'] + + @retrier + def get_balances(self) -> dict: + if self._CONF['dry_run']: + return {} + + try: + balances = self._API.fetch_balance() + # Remove additional info from ccxt results + balances.pop("info", None) + balances.pop("free", None) + balances.pop("total", None) + balances.pop("used", None) + + return balances + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get balance due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_tickers(self) -> Dict: + try: + return self._API.fetch_tickers() + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._API.name} does not support fetching tickers in batch.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: + if refresh or pair not in self._CACHED_TICKER.keys(): try: - _CACHED_TICKER[pair] = { - 'bid': float(data['bid']), - 'ask': float(data['ask']), - } - except KeyError: - logger.debug("Could not cache ticker data for %s", pair) + data = self._API.fetch_ticker(pair) + try: + self._CACHED_TICKER[pair] = { + 'bid': float(data['bid']), + 'ask': float(data['ask']), + } + except KeyError: + logger.debug("Could not cache ticker data for %s", pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + else: + logger.info("returning cached ticker-data for %s", pair) + return self._CACHED_TICKER[pair] + + @retrier + def get_ticker_history(self, pair: str, tick_interval: str, + since_ms: Optional[int] = None) -> List[Dict]: + try: + # last item should be in the time interval [now - tick_interval, now] + till_time_ms = arrow.utcnow().shift( + minutes=-constants.TICKER_INTERVAL_MINUTES[tick_interval] + ).timestamp * 1000 + # it looks as if some exchanges return cached data + # and they update it one in several minute, so 10 mins interval + # is necessary to skeep downloading of an empty array when all + # chached data was already downloaded + till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) + + data: List[Dict[Any, Any]] = [] + while not since_ms or since_ms < till_time_ms: + data_part = self._API.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + data_part = sorted(data_part, key=lambda x: x[0]) + + if not data_part: + break + + logger.debug('Downloaded data for %s time range [%s, %s]', + pair, + arrow.get(data_part[0][0] / 1000).format(), + arrow.get(data_part[-1][0] / 1000).format()) + + data.extend(data_part) + since_ms = data[-1][0] + 1 + return data + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._API.name} does not support fetching historical candlestick data.' + f'Message: {e}') except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + + @retrier + def cancel_order(self, order_id: str, pair: str) -> None: + if self._CONF['dry_run']: + return + + try: + return self._API.cancel_order(order_id, pair) + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not cancel order. Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') except ccxt.BaseError as e: raise OperationalException(e) - else: - logger.info("returning cached ticker-data for %s", pair) - return _CACHED_TICKER[pair] + @retrier + def get_order(self, order_id: str, pair: str) -> Dict: + if self._CONF['dry_run']: + order = self._DRY_RUN_OPEN_ORDERS[order_id] + order.update({ + 'id': order_id + }) + return order + try: + return self._API.fetch_order(order_id, pair) + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not get order. Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) -@retrier -def get_ticker_history(pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: - try: - # last item should be in the time interval [now - tick_interval, now] - till_time_ms = arrow.utcnow().shift( - minutes=-constants.TICKER_INTERVAL_MINUTES[tick_interval] - ).timestamp * 1000 - # it looks as if some exchanges return cached data - # and they update it one in several minute, so 10 mins interval - # is necessary to skeep downloading of an empty array when all - # chached data was already downloaded - till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) + @retrier + def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + if self._CONF['dry_run']: + return [] + if not self.exchange_has('fetchMyTrades'): + return [] + try: + my_trades = self._API.fetch_my_trades(pair, since.timestamp()) + matched_trades = [trade for trade in my_trades if trade['order'] == order_id] - data: List[Dict[Any, Any]] = [] - while not since_ms or since_ms < till_time_ms: - data_part = _API.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + return matched_trades - # Because some exchange sort Tickers ASC and other DESC. - # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) - # when GDAX returns a list of tickers DESC (newest first, oldest last) - data_part = sorted(data_part, key=lambda x: x[0]) + except ccxt.NetworkError as e: + raise TemporaryError( + f'Could not get trades due to networking error. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) - if not data_part: - break + def get_pair_detail_url(self, pair: str) -> str: + try: + url_base = self._API.urls.get('www') + base, quote = pair.split('/') - logger.debug('Downloaded data for %s time range [%s, %s]', - pair, - arrow.get(data_part[0][0] / 1000).format(), - arrow.get(data_part[-1][0] / 1000).format()) + return url_base + _EXCHANGE_URLS[self._API.id].format(base=base, quote=quote) + except KeyError: + logger.warning('Could not get exchange url for %s', self.get_name()) + return "" - data.extend(data_part) - since_ms = data[-1][0] + 1 + @retrier + def get_markets(self) -> List[dict]: + try: + return self._API.fetch_markets() + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load markets due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) - return data - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {_API.name} does not support fetching historical candlestick data.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + @retrier + def get_fee(self, symbol='ETH/BTC', type='', side='', amount=1, + price=1, taker_or_maker='maker') -> float: + try: + # validate that markets are loaded before trying to get fee + if self._API.markets is None or len(self._API.markets) == 0: + self._API.load_markets() + return self._API.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + price=price, takerOrMaker=taker_or_maker)['rate'] + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) -@retrier -def cancel_order(order_id: str, pair: str) -> None: - if _CONF['dry_run']: - return - - try: - return _API.cancel_order(order_id, pair) - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not cancel order. Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -@retrier -def get_order(order_id: str, pair: str) -> Dict: - if _CONF['dry_run']: - order = _DRY_RUN_OPEN_ORDERS[order_id] - order.update({ - 'id': order_id - }) - return order - try: - return _API.fetch_order(order_id, pair) - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not get order. Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -@retrier -def get_trades_for_order(order_id: str, pair: str, since: datetime) -> List: - if _CONF['dry_run']: - return [] - if not exchange_has('fetchMyTrades'): - return [] - try: - my_trades = _API.fetch_my_trades(pair, since.timestamp()) - matched_trades = [trade for trade in my_trades if trade['order'] == order_id] - - return matched_trades - - except ccxt.NetworkError as e: - raise TemporaryError( - f'Could not get trades due to networking error. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -def get_pair_detail_url(pair: str) -> str: - try: - url_base = _API.urls.get('www') - base, quote = pair.split('/') - - return url_base + _EXCHANGE_URLS[_API.id].format(base=base, quote=quote) - except KeyError: - logger.warning('Could not get exchange url for %s', get_name()) - return "" - - -@retrier -def get_markets() -> List[dict]: - try: - return _API.fetch_markets() - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load markets due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -def get_name() -> str: - return _API.name - - -def get_id() -> str: - return _API.id - - -@retrier -def get_fee(symbol='ETH/BTC', type='', side='', amount=1, - price=1, taker_or_maker='maker') -> float: - try: + def get_amount_lots(self, pair: str, amount: float) -> float: + """ + get buyable amount rounding, .. + """ # validate that markets are loaded before trying to get fee - if _API.markets is None or len(_API.markets) == 0: - _API.load_markets() - - return _API.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, - price=price, takerOrMaker=taker_or_maker)['rate'] - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - -def get_amount_lots(pair: str, amount: float) -> float: - """ - get buyable amount rounding, .. - """ - # validate that markets are loaded before trying to get fee - if not _API.markets: - _API.load_markets() - return _API.amount_to_lots(pair, amount) + if not self._API.markets: + self._API.load_markets() + return self._API.amount_to_lots(pair, amount) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 157de862f..5f69b8115 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -14,11 +14,11 @@ import requests from cachetools import TTLCache, cached from freqtrade import ( - DependencyException, OperationalException, TemporaryError, - exchange, persistence, __version__, + DependencyException, OperationalException, TemporaryError, persistence, __version__, ) from freqtrade import constants from freqtrade.analyze import Analyze +from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade from freqtrade.rpc.rpc_manager import RPCManager @@ -66,7 +66,7 @@ class FreqtradeBot(object): # Initialize all modules persistence.init(self.config) - exchange.init(self.config) + self.exchange = Exchange(self.config) # Set initial application state initial_state = self.config.get('initial_state') @@ -186,13 +186,13 @@ class FreqtradeBot(object): :return: List of pairs """ - if not exchange.exchange_has('fetchTickers'): + if not self.exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' ) - tickers = exchange.get_tickers() + tickers = self.exchange.get_tickers() # check length so that we make sure that '/' is actually in the string tickers = [v for k, v in tickers.items() if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] @@ -210,7 +210,7 @@ class FreqtradeBot(object): black_listed """ sanitized_whitelist = whitelist - markets = exchange.get_markets() + markets = self.exchange.get_markets() markets = [m for m in markets if m['quote'] == self.config['stake_currency']] known_pairs = set() @@ -255,7 +255,7 @@ class FreqtradeBot(object): interval = self.analyze.get_ticker_interval() stake_currency = self.config['stake_currency'] fiat_currency = self.config['fiat_display_currency'] - exc_name = exchange.get_name() + exc_name = self.exchange.get_name() logger.info( 'Checking buy signals to create a new trade with stake_amount: %f ...', @@ -263,7 +263,7 @@ class FreqtradeBot(object): ) whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) # Check if stake_amount is fulfilled - if exchange.get_balance(stake_currency) < stake_amount: + if self.exchange.get_balance(stake_currency) < stake_amount: raise DependencyException( f'stake amount is not fulfilled (currency={stake_currency})') @@ -278,19 +278,19 @@ class FreqtradeBot(object): # Pick pair based on buy signals for _pair in whitelist: - (buy, sell) = self.analyze.get_signal(_pair, interval) + (buy, sell) = self.analyze.get_signal(self.exchange, _pair, interval) if buy and not sell: pair = _pair break else: return False pair_s = pair.replace('_', '/') - pair_url = exchange.get_pair_detail_url(pair) + pair_url = self.exchange.get_pair_detail_url(pair) # Calculate amount - buy_limit = self.get_target_bid(exchange.get_ticker(pair)) + buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) amount = stake_amount / buy_limit - order_id = exchange.buy(pair, buy_limit, amount)['id'] + order_id = self.exchange.buy(pair, buy_limit, amount)['id'] stake_amount_fiat = self.fiat_converter.convert_amount( stake_amount, @@ -305,7 +305,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ {stake_currency}, {stake_amount_fiat:.3f} {fiat_currency})`""" ) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL - fee = exchange.get_fee(symbol=pair, taker_or_maker='maker') + fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') trade = Trade( pair=pair, stake_amount=stake_amount, @@ -315,7 +315,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ open_rate=buy_limit, open_rate_requested=buy_limit, open_date=datetime.utcnow(), - exchange=exchange.get_id(), + exchange=self.exchange.get_id(), open_order_id=order_id ) Trade.session.add(trade) @@ -348,7 +348,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ if trade.open_order_id: # Update trade with order values logger.info('Found open order for %s', trade) - order = exchange.get_order(trade.open_order_id, trade.pair) + order = self.exchange.get_order(trade.open_order_id, trade.pair) # Try update amount (binance-fix) try: new_amount = self.get_real_amount(trade, order) @@ -372,7 +372,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ def get_real_amount(self, trade: Trade, order: Dict) -> float: """ Get real amount for the trade - Necessary for exchanges which charge fees in base currency (e.g. binance) + Necessary for self.exchanges which charge fees in base currency (e.g. binance) """ order_amount = order['amount'] # Only run for closed orders @@ -388,7 +388,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ return new_amount # Fallback to Trades - trades = exchange.get_trades_for_order(trade.open_order_id, trade.pair, trade.open_date) + trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, + trade.open_date) if len(trades) == 0: logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) @@ -420,7 +421,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ raise ValueError(f'attempt to handle closed trade: {trade}') logger.debug('Handling %s ...', trade) - current_rate = exchange.get_ticker(trade.pair)['bid'] + current_rate = self.exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) @@ -449,7 +450,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ # updated via /forcesell in a different thread. if not trade.open_order_id: continue - order = exchange.get_order(trade.open_order_id, trade.pair) + order = self.exchange.get_order(trade.open_order_id, trade.pair) except requests.exceptions.RequestException: logger.info( 'Cannot query order for %s due to %s', @@ -475,7 +476,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ :return: True if order was fully cancelled """ pair_s = trade.pair.replace('_', '/') - exchange.cancel_order(trade.open_order_id, trade.pair) + self.exchange.cancel_order(trade.open_order_id, trade.pair) if order['remaining'] == order['amount']: # if trade is not partially completed, just delete the trade Trade.session.delete(trade) @@ -502,7 +503,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ pair_s = trade.pair.replace('_', '/') if order['remaining'] == order['amount']: # if trade is not partially completed, just cancel the trade - exchange.cancel_order(trade.open_order_id, trade.pair) + self.exchange.cancel_order(trade.open_order_id, trade.pair) trade.close_rate = None trade.close_profit = None trade.close_date = None @@ -525,15 +526,15 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ exc = trade.exchange pair = trade.pair # Execute sell and update trade record - order_id = exchange.sell(str(trade.pair), limit, trade.amount)['id'] + order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id'] trade.open_order_id = order_id trade.close_rate_requested = limit fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) profit_trade = trade.calc_profit(rate=limit) - current_rate = exchange.get_ticker(trade.pair)['bid'] + current_rate = self.exchange.get_ticker(trade.pair)['bid'] profit = trade.calc_profit_percent(limit) - pair_url = exchange.get_pair_detail_url(trade.pair) + pair_url = self.exchange.get_pair_detail_url(trade.pair) gain = "profit" if fmt_exp_profit > 0 else "loss" message = f"*{exc}:* Selling\n" \ diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 867e8c7dc..3e2fb4554 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -8,7 +8,7 @@ from typing import Optional, List, Dict, Tuple, Any import arrow from freqtrade import misc, constants -from freqtrade.exchange import get_ticker_history +from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange logger = logging.getLogger(__name__) @@ -183,6 +183,7 @@ def load_cached_data_for_updating(filename: str, def download_backtesting_testdata(datadir: str, + exchange: Exchange, pair: str, tick_interval: str = '5m', timerange: Optional[TimeRange] = None) -> None: @@ -216,7 +217,8 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - new_data = get_ticker_history(pair=pair, tick_interval=tick_interval, since_ms=since_ms) + new_data = exchange.get_ticker_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 23bd7741a..58b552237 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -14,7 +14,7 @@ from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize -from freqtrade import exchange +from freqtrade.exchange import Exchange from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -61,7 +61,7 @@ class Backtesting(object): self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True - exchange.init(self.config) + self.exchange = Exchange(self.config) @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -130,7 +130,7 @@ class Backtesting(object): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) - fee = exchange.get_fee() + fee = self.exchange.get_fee() trade = Trade( open_rate=buy_row.close, open_date=buy_row.date, @@ -256,7 +256,7 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') for pair in pairs: - data[pair] = exchange.get_ticker_history(pair, self.ticker_interval) + data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') From dea26fadfe5499d5fd66391476864adc6bf71050 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 13:09:23 +0200 Subject: [PATCH 077/239] move init_ccxt to class --- freqtrade/exchange/__init__.py | 53 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 5ac5f8a0b..0d303fa4a 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -40,32 +40,6 @@ def retrier(f): return wrapper -def init_ccxt(exchange_config: dict) -> ccxt.Exchange: - """ - Initialize ccxt with given config and return valid - ccxt instance. - :param config: config to use - :return: ccxt - """ - # Find matching class for the given exchange name - name = exchange_config['name'] - - if name not in ccxt.exchanges: - raise OperationalException(f'Exchange {name} is not supported') - try: - api = getattr(ccxt, name.lower())({ - 'apiKey': exchange_config.get('key'), - 'secret': exchange_config.get('secret'), - 'password': exchange_config.get('password'), - 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': True, - }) - except (KeyError, AttributeError): - raise OperationalException(f'Exchange {name} is not supported') - - return api - - class Exchange(object): # Current selected exchange @@ -92,13 +66,38 @@ class Exchange(object): logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] - self._API = init_ccxt(exchange_config) + self._API = self._init_ccxt(exchange_config) logger.info('Using Exchange "%s"', self.get_name()) # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) + def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange: + """ + Initialize ccxt with given config and return valid + ccxt instance. + :param config: config to use + :return: ccxt + """ + # Find matching class for the given exchange name + name = exchange_config['name'] + + if name not in ccxt.exchanges: + raise OperationalException(f'Exchange {name} is not supported') + try: + api = getattr(ccxt, name.lower())({ + 'apiKey': exchange_config.get('key'), + 'secret': exchange_config.get('secret'), + 'password': exchange_config.get('password'), + 'uid': exchange_config.get('uid', ''), + 'enableRateLimit': True, + }) + except (KeyError, AttributeError): + raise OperationalException(f'Exchange {name} is not supported') + + return api + def get_name(self) -> str: return self._API.name From a159db68637bec69e4438c272d76e4ce1fa36eb2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 13:32:05 +0200 Subject: [PATCH 078/239] get_exchange --- freqtrade/tests/conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 1311687b7..ed900e6bb 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -13,6 +13,7 @@ from telegram import Chat, Message, Update from freqtrade.analyze import Analyze from freqtrade import constants +from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -26,6 +27,17 @@ def log_has(line, logs): False) +def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: + + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + if api_mock: + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + else: + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) + exchange = Exchange(config) + return exchange + + # Functions for recurrent object patching def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ From 67d345bc0859fac1184fae0552b9e967dd279ca6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 19:54:51 +0200 Subject: [PATCH 079/239] fix tests for objectify exchange --- freqtrade/tests/test_acl_pair.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 07375260e..608e6a4a1 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -32,7 +32,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets): freqtradebot = tt.get_patched_freqtradebot(mocker, conf) - mocker.patch('freqtrade.freqtradebot.exchange.get_markets', markets) + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) refreshedwhitelist = freqtradebot._refresh_whitelist( conf['exchange']['pair_whitelist'] + ['XXX/BTC'] ) @@ -46,7 +46,7 @@ def test_refresh_whitelist(mocker, markets): conf = whitelist_conf() freqtradebot = tt.get_patched_freqtradebot(mocker, conf) - mocker.patch('freqtrade.freqtradebot.exchange.get_markets', markets) + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) refreshedwhitelist = freqtradebot._refresh_whitelist(conf['exchange']['pair_whitelist']) # List ordered by BaseVolume @@ -59,7 +59,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers): conf = whitelist_conf() freqtradebot = tt.get_patched_freqtradebot(mocker, conf) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', get_markets=markets, get_tickers=tickers, exchange_has=MagicMock(return_value=True) @@ -78,7 +78,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers): def test_refresh_whitelist_dynamic_empty(mocker, markets_empty): conf = whitelist_conf() freqtradebot = tt.get_patched_freqtradebot(mocker, conf) - mocker.patch('freqtrade.freqtradebot.exchange.get_markets', markets_empty) + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets_empty) # argument: use the whitelist dynamically by exchange-volume whitelist = [] From 68f6423d39519cf1f30dcb9baab49e82efba4f80 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 20:13:39 +0200 Subject: [PATCH 080/239] fix most tests --- freqtrade/tests/exchange/test_exchange.py | 283 ++++++++++------------ 1 file changed, 130 insertions(+), 153 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 97a723929..2479d26b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -8,28 +8,14 @@ from unittest.mock import MagicMock, PropertyMock import ccxt import pytest -import freqtrade.exchange as exchange from freqtrade import OperationalException, DependencyException, TemporaryError -from freqtrade.exchange import (init, validate_pairs, buy, sell, get_balance, get_balances, - get_ticker, get_ticker_history, cancel_order, get_name, get_fee, - get_id, get_pair_detail_url, get_amount_lots) -from freqtrade.tests.conftest import log_has - -API_INIT = False - - -def maybe_init_api(conf, mocker, force=False): - global API_INIT - if force or not API_INIT: - mocker.patch('freqtrade.exchange.validate_pairs', - side_effect=lambda s: True) - init(config=conf) - API_INIT = True +from freqtrade.exchange import Exchange, API_RETRY_COUNT +from freqtrade.tests.conftest import log_has, get_patched_exchange def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - maybe_init_api(default_conf, mocker, True) + get_patched_exchange(mocker, default_conf) assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) @@ -39,7 +25,7 @@ def test_init_exception(default_conf): with pytest.raises( OperationalException, match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): - init(config=default_conf) + Exchange(default_conf) def test_validate_pairs(default_conf, mocker): @@ -50,18 +36,17 @@ def test_validate_pairs(default_conf, mocker): id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - validate_pairs(default_conf['exchange']['pair_whitelist']) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + Exchange(default_conf) def test_validate_pairs_not_available(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={}) - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + with pytest.raises(OperationalException, match=r'not available'): - validate_pairs(default_conf['exchange']['pair_whitelist']) + Exchange(default_conf) def test_validate_pairs_not_compatible(default_conf, mocker): @@ -71,25 +56,24 @@ def test_validate_pairs_not_compatible(default_conf, mocker): }) conf = deepcopy(default_conf) conf['stake_currency'] = 'ETH' - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', conf) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + with pytest.raises(OperationalException, match=r'not compatible'): - validate_pairs(conf['exchange']['pair_whitelist']) + Exchange(conf) def test_validate_pairs_exception(default_conf, mocker, caplog): caplog.set_level(logging.INFO) api_mock = MagicMock() api_mock.name = 'Binance' - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) + exchange = Exchange(default_conf) api_mock.load_markets = MagicMock(return_value={}) with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): - validate_pairs(default_conf['exchange']['pair_whitelist']) + exchange.validate_pairs(default_conf['exchange']['pair_whitelist']) api_mock.load_markets = MagicMock(side_effect=ccxt.BaseError()) - validate_pairs(default_conf['exchange']['pair_whitelist']) + exchange.validate_pairs(default_conf['exchange']['pair_whitelist']) assert log_has('Unable to validate pairs (assuming they are correct). Reason: ', caplog.record_tuples) @@ -100,21 +84,20 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): conf['stake_currency'] = 'ETH' api_mock = MagicMock() api_mock.name = 'binance' - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', conf) + exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises( OperationalException, match=r'Pair ETH/BTC not compatible with stake_currency: ETH' ): - validate_pairs(default_conf['exchange']['pair_whitelist']) + exchange.validate_pairs(default_conf['exchange']['pair_whitelist']) def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + exchange = get_patched_exchange(mocker, default_conf) - order = buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) assert 'id' in order assert 'dry_run_buy_' in order['id'] @@ -128,12 +111,10 @@ def test_buy_prod(default_conf, mocker): 'foo': 'bar' } }) - mocker.patch('freqtrade.exchange._API', api_mock) - default_conf['dry_run'] = False - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + exchange = get_patched_exchange(mocker, default_conf, api_mock) - order = buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) assert 'id' in order assert 'info' in order assert order['id'] == order_id @@ -141,30 +122,30 @@ def test_buy_prod(default_conf, mocker): # test exception handling with pytest.raises(DependencyException): api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InsufficientFunds) - mocker.patch('freqtrade.exchange._API', api_mock) - buy(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(DependencyException): api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InvalidOrder) - mocker.patch('freqtrade.exchange._API', api_mock) - buy(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(TemporaryError): api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.NetworkError) - mocker.patch('freqtrade.exchange._API', api_mock) - buy(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(OperationalException): api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) - buy(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.buy(pair='ETH/BTC', rate=200, amount=1) def test_sell_dry_run(default_conf, mocker): default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + exchange = get_patched_exchange(mocker, default_conf) - order = sell(pair='ETH/BTC', rate=200, amount=1) + order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) assert 'id' in order assert 'dry_run_sell_' in order['id'] @@ -178,12 +159,11 @@ def test_sell_prod(default_conf, mocker): 'foo': 'bar' } }) - mocker.patch('freqtrade.exchange._API', api_mock) - default_conf['dry_run'] = False - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - order = sell(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) assert 'id' in order assert 'info' in order assert order['id'] == order_id @@ -191,53 +171,52 @@ def test_sell_prod(default_conf, mocker): # test exception handling with pytest.raises(DependencyException): api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InsufficientFunds) - mocker.patch('freqtrade.exchange._API', api_mock) - sell(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(DependencyException): api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InvalidOrder) - mocker.patch('freqtrade.exchange._API', api_mock) - sell(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(TemporaryError): api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.NetworkError) - mocker.patch('freqtrade.exchange._API', api_mock) - sell(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(OperationalException): api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) - sell(pair='ETH/BTC', rate=200, amount=1) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.sell(pair='ETH/BTC', rate=200, amount=1) def test_get_balance_dry_run(default_conf, mocker): default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert get_balance(currency='BTC') == 999.9 + exchange = get_patched_exchange(mocker, default_conf) + assert exchange.get_balance(currency='BTC') == 999.9 def test_get_balance_prod(default_conf, mocker): api_mock = MagicMock() api_mock.fetch_balance = MagicMock(return_value={'BTC': {'free': 123.4}}) - mocker.patch('freqtrade.exchange._API', api_mock) - default_conf['dry_run'] = False - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert get_balance(currency='BTC') == 123.4 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + assert exchange.get_balance(currency='BTC') == 123.4 with pytest.raises(OperationalException): api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) - get_balance(currency='BTC') + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + exchange.get_balance(currency='BTC') def test_get_balances_dry_run(default_conf, mocker): default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - - assert get_balances() == {} + exchange = get_patched_exchange(mocker, default_conf) + assert exchange.get_balances() == {} def test_get_balances_prod(default_conf, mocker): @@ -253,33 +232,29 @@ def test_get_balances_prod(default_conf, mocker): '2ST': balance_item, '3ST': balance_item }) - mocker.patch('freqtrade.exchange._API', api_mock) - default_conf['dry_run'] = False - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - - assert len(get_balances()) == 3 - assert get_balances()['1ST']['free'] == 10.0 - assert get_balances()['1ST']['total'] == 10.0 - assert get_balances()['1ST']['used'] == 0.0 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert len(exchange.get_balances()) == 3 + assert exchange.get_balances()['1ST']['free'] == 10.0 + assert exchange.get_balances()['1ST']['total'] == 10.0 + assert exchange.get_balances()['1ST']['used'] == 0.0 with pytest.raises(TemporaryError): api_mock.fetch_balance = MagicMock(side_effect=ccxt.NetworkError) - mocker.patch('freqtrade.exchange._API', api_mock) - get_balances() - assert api_mock.fetch_balance.call_count == exchange.API_RETRY_COUNT + 1 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_balances() + assert api_mock.fetch_balance.call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) - get_balances() + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_balances() assert api_mock.fetch_balance.call_count == 1 # This test is somewhat redundant with # test_exchange_bittrex.py::test_exchange_bittrex_get_ticker def test_get_ticker(default_conf, mocker): - maybe_init_api(default_conf, mocker) api_mock = MagicMock() tick = { 'symbol': 'ETH/BTC', @@ -288,10 +263,9 @@ def test_get_ticker(default_conf, mocker): 'last': 0.0001, } api_mock.fetch_ticker = MagicMock(return_value=tick) - mocker.patch('freqtrade.exchange._API', api_mock) - + exchange = get_patched_exchange(mocker, default_conf, api_mock) # retrieve original ticker - ticker = get_ticker(pair='ETH/BTC') + ticker = exchange.get_ticker(pair='ETH/BTC') assert ticker['bid'] == 0.00001098 assert ticker['ask'] == 0.00001099 @@ -304,11 +278,11 @@ def test_get_ticker(default_conf, mocker): 'last': 42, } api_mock.fetch_ticker = MagicMock(return_value=tick) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) # if not caching the result we should get the same ticker # if not fetching a new result we should get the cached ticker - ticker = get_ticker(pair='ETH/BTC') + ticker = exchange.get_ticker(pair='ETH/BTC') assert api_mock.fetch_ticker.call_count == 1 assert ticker['bid'] == 0.5 @@ -320,22 +294,22 @@ def test_get_ticker(default_conf, mocker): # Test caching api_mock.fetch_ticker = MagicMock() - get_ticker(pair='ETH/BTC', refresh=False) + exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 with pytest.raises(TemporaryError): # test retrier api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) - mocker.patch('freqtrade.exchange._API', api_mock) - get_ticker(pair='ETH/BTC', refresh=True) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_ticker(pair='ETH/BTC', refresh=True) with pytest.raises(OperationalException): api_mock.fetch_ticker = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) - get_ticker(pair='ETH/BTC', refresh=True) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_ticker(pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) - mocker.patch('freqtrade.exchange._API', api_mock) - get_ticker(pair='ETH/BTC', refresh=True) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_ticker(pair='ETH/BTC', refresh=True) def make_fetch_ohlcv_mock(data): @@ -361,10 +335,10 @@ def test_get_ticker_history(default_conf, mocker): ] type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) # retrieve original ticker - ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1511686200000 assert ticks[0][1] == 1 assert ticks[0][2] == 2 @@ -384,9 +358,9 @@ def test_get_ticker_history(default_conf, mocker): ] ] api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick)) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) - ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1511686210000 assert ticks[0][1] == 6 assert ticks[0][2] == 7 @@ -396,15 +370,15 @@ def test_get_ticker_history(default_conf, mocker): with pytest.raises(TemporaryError): # test retrier api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NetworkError) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) # new symbol to get around cache - get_ticker_history('ABCD/BTC', default_conf['ticker_interval']) + exchange.get_ticker_history('ABCD/BTC', default_conf['ticker_interval']) with pytest.raises(OperationalException): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) # new symbol to get around cache - get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) + exchange.get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) def test_get_ticker_history_sort(default_conf, mocker): @@ -426,10 +400,11 @@ def test_get_ticker_history_sort(default_conf, mocker): ] type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - mocker.patch('freqtrade.exchange._API', api_mock) + + exchange = get_patched_exchange(mocker, default_conf, api_mock) # Test the ticker history sort - ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -460,10 +435,9 @@ def test_get_ticker_history_sort(default_conf, mocker): ] type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - mocker.patch('freqtrade.exchange._API', api_mock) - + exchange = get_patched_exchange(mocker, default_conf, api_mock) # Test the ticker history sort - ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 @@ -481,114 +455,115 @@ def test_get_ticker_history_sort(default_conf, mocker): def test_cancel_order_dry_run(default_conf, mocker): default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - - assert cancel_order(order_id='123', pair='TKN/BTC') is None + exchange = get_patched_exchange(mocker, default_conf) + assert exchange.cancel_order(order_id='123', pair='TKN/BTC') is None # Ensure that if not dry_run, we should call API def test_cancel_order(default_conf, mocker): default_conf['dry_run'] = False - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + # mocker.patch.dict('freqtrade.exchange.._CONF', default_conf) api_mock = MagicMock() api_mock.cancel_order = MagicMock(return_value=123) - mocker.patch('freqtrade.exchange._API', api_mock) - assert cancel_order(order_id='_', pair='TKN/BTC') == 123 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == 123 with pytest.raises(TemporaryError): api_mock.cancel_order = MagicMock(side_effect=ccxt.NetworkError) - mocker.patch('freqtrade.exchange._API', api_mock) - cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == exchange.API_RETRY_COUNT + 1 + + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.cancel_order(order_id='_', pair='TKN/BTC') + assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 with pytest.raises(DependencyException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder) - mocker.patch('freqtrade.exchange._API', api_mock) - cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == exchange.API_RETRY_COUNT + 1 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.cancel_order(order_id='_', pair='TKN/BTC') + assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): api_mock.cancel_order = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) - cancel_order(order_id='_', pair='TKN/BTC') + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == 1 def test_get_order(default_conf, mocker): default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + mocker.patch.dict('freqtrade.exchange.Exchange._CONF', default_conf) order = MagicMock() order.myid = 123 + exchange = Exchange(default_conf) exchange._DRY_RUN_OPEN_ORDERS['X'] = order print(exchange.get_order('X', 'TKN/BTC')) assert exchange.get_order('X', 'TKN/BTC').myid == 123 default_conf['dry_run'] = False - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + mocker.patch.dict('freqtrade.exchange.Exchange._CONF', default_conf) api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value=456) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.get_order('X', 'TKN/BTC') == 456 with pytest.raises(TemporaryError): api_mock.fetch_order = MagicMock(side_effect=ccxt.NetworkError) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == exchange.API_RETRY_COUNT + 1 + assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 with pytest.raises(DependencyException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == exchange.API_RETRY_COUNT + 1 + assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): api_mock.fetch_order = MagicMock(side_effect=ccxt.BaseError) - mocker.patch('freqtrade.exchange._API', api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == 1 def test_get_name(default_conf, mocker): - mocker.patch('freqtrade.exchange.validate_pairs', + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', side_effect=lambda s: True) default_conf['exchange']['name'] = 'binance' - init(default_conf) + exchange = Exchange(default_conf) - assert get_name() == 'Binance' + assert exchange.get_name() == 'Binance' def test_get_id(default_conf, mocker): - mocker.patch('freqtrade.exchange.validate_pairs', + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', side_effect=lambda s: True) default_conf['exchange']['name'] = 'binance' - init(default_conf) - - assert get_id() == 'binance' + exchange = Exchange(default_conf) + assert exchange.get_id() == 'binance' def test_get_pair_detail_url(default_conf, mocker): - mocker.patch('freqtrade.exchange.validate_pairs', + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', side_effect=lambda s: True) default_conf['exchange']['name'] = 'binance' - init(default_conf) + # exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = Exchange(default_conf) - url = get_pair_detail_url('TKN/ETH') + url = exchange.get_pair_detail_url('TKN/ETH') assert 'TKN' in url assert 'ETH' in url - url = get_pair_detail_url('LOOONG/BTC') + url = exchange.get_pair_detail_url('LOOONG/BTC') assert 'LOOONG' in url assert 'BTC' in url default_conf['exchange']['name'] = 'bittrex' - init(default_conf) + exchange = Exchange(default_conf) - url = get_pair_detail_url('TKN/ETH') + url = exchange.get_pair_detail_url('TKN/ETH') assert 'TKN' in url assert 'ETH' in url - url = get_pair_detail_url('LOOONG/BTC') + url = exchange.get_pair_detail_url('LOOONG/BTC') assert 'LOOONG' in url assert 'BTC' in url @@ -601,12 +576,14 @@ def test_get_fee(default_conf, mocker): 'rate': 0.025, 'cost': 0.05 }) - mocker.patch('freqtrade.exchange._API', api_mock) - assert get_fee() == 0.025 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + assert exchange.get_fee() == 0.025 def test_get_amount_lots(default_conf, mocker): api_mock = MagicMock() api_mock.amount_to_lots = MagicMock(return_value=1.0) - mocker.patch('freqtrade.exchange._API', api_mock) - assert get_amount_lots('LTC/BTC', 1.54) == 1 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + assert exchange.get_amount_lots('LTC/BTC', 1.54) == 1 From 495f15f13c8ccf96e0409c1b49e5bdf659240526 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 20:35:52 +0200 Subject: [PATCH 081/239] fix exchange tests --- freqtrade/tests/exchange/test_exchange.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 2479d26b0..9c23b36af 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -65,15 +65,18 @@ def test_validate_pairs_not_compatible(default_conf, mocker): def test_validate_pairs_exception(default_conf, mocker, caplog): caplog.set_level(logging.INFO) api_mock = MagicMock() - api_mock.name = 'Binance' - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) - exchange = Exchange(default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_name', MagicMock(return_value='Binance')) + api_mock.load_markets = MagicMock(return_value={}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): - exchange.validate_pairs(default_conf['exchange']['pair_whitelist']) + Exchange(default_conf) api_mock.load_markets = MagicMock(side_effect=ccxt.BaseError()) - exchange.validate_pairs(default_conf['exchange']['pair_whitelist']) + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + Exchange(default_conf) assert log_has('Unable to validate pairs (assuming they are correct). Reason: ', caplog.record_tuples) @@ -83,14 +86,14 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): conf = deepcopy(default_conf) conf['stake_currency'] = 'ETH' api_mock = MagicMock() - api_mock.name = 'binance' - exchange = get_patched_exchange(mocker, default_conf, api_mock) + api_mock.name = MagicMock(return_value='binance') + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) with pytest.raises( OperationalException, match=r'Pair ETH/BTC not compatible with stake_currency: ETH' ): - exchange.validate_pairs(default_conf['exchange']['pair_whitelist']) + Exchange(conf) def test_buy_dry_run(default_conf, mocker): From e8ab76f55bd771109f6418b75c1238406c216782 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 20:37:10 +0200 Subject: [PATCH 082/239] fix small in tests --- freqtrade/tests/exchange/test_exchange.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 9c23b36af..c66116ae0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -493,16 +493,14 @@ def test_cancel_order(default_conf, mocker): def test_get_order(default_conf, mocker): default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.exchange.Exchange._CONF', default_conf) order = MagicMock() order.myid = 123 - exchange = Exchange(default_conf) + exchange = get_patched_exchange(mocker, default_conf) exchange._DRY_RUN_OPEN_ORDERS['X'] = order print(exchange.get_order('X', 'TKN/BTC')) assert exchange.get_order('X', 'TKN/BTC').myid == 123 default_conf['dry_run'] = False - mocker.patch.dict('freqtrade.exchange.Exchange._CONF', default_conf) api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value=456) exchange = get_patched_exchange(mocker, default_conf, api_mock) From 082b6077e9390c8af983738d45749d7af992e7eb Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 20:48:12 +0200 Subject: [PATCH 083/239] Fix tests analyze --- freqtrade/tests/test_analyze.py | 48 ++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index e8d0816aa..89dac21c6 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -14,7 +14,7 @@ from pandas import DataFrame from freqtrade.analyze import Analyze, SignalType from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has +from freqtrade.tests.conftest import log_has, get_patched_exchange # Avoid to reinit the same object again and again _ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) @@ -66,16 +66,16 @@ def test_populates_sell_trend(result): assert 'sell' in dataframe.columns -def test_returns_latest_buy_signal(mocker): - mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) - +def test_returns_latest_buy_signal(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) mocker.patch.multiple( 'freqtrade.analyze.Analyze', analyze_ticker=MagicMock( return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) ) ) - assert _ANALYZE.get_signal('ETH/BTC', '5m') == (True, False) + assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) mocker.patch.multiple( 'freqtrade.analyze.Analyze', @@ -83,11 +83,12 @@ def test_returns_latest_buy_signal(mocker): return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) ) ) - assert _ANALYZE.get_signal('ETH/BTC', '5m') == (False, True) + assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) -def test_returns_latest_sell_signal(mocker): - mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) +def test_returns_latest_sell_signal(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) mocker.patch.multiple( 'freqtrade.analyze.Analyze', analyze_ticker=MagicMock( @@ -95,7 +96,7 @@ def test_returns_latest_sell_signal(mocker): ) ) - assert _ANALYZE.get_signal('ETH/BTC', '5m') == (False, True) + assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) mocker.patch.multiple( 'freqtrade.analyze.Analyze', @@ -103,45 +104,49 @@ def test_returns_latest_sell_signal(mocker): return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) ) ) - assert _ANALYZE.get_signal('ETH/BTC', '5m') == (True, False) + assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) def test_get_signal_empty(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch('freqtrade.analyze.get_ticker_history', return_value=None) - assert (False, False) == _ANALYZE.get_signal('foo', default_conf['ticker_interval']) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) + exchange = get_patched_exchange(mocker, default_conf) + assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) assert log_has('Empty ticker history for pair foo', caplog.record_tuples) def test_get_signal_exception_valueerror(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch('freqtrade.analyze.get_ticker_history', return_value=1) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) mocker.patch.multiple( 'freqtrade.analyze.Analyze', analyze_ticker=MagicMock( side_effect=ValueError('xyz') ) ) - assert (False, False) == _ANALYZE.get_signal('foo', default_conf['ticker_interval']) + assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) def test_get_signal_empty_dataframe(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch('freqtrade.analyze.get_ticker_history', return_value=1) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) mocker.patch.multiple( 'freqtrade.analyze.Analyze', analyze_ticker=MagicMock( return_value=DataFrame([]) ) ) - assert (False, False) == _ANALYZE.get_signal('xyz', default_conf['ticker_interval']) + assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) def test_get_signal_old_dataframe(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch('freqtrade.analyze.get_ticker_history', return_value=1) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) # FIX: The get_signal function has hardcoded 10, which we must inturn hardcode oldtime = arrow.utcnow() - datetime.timedelta(minutes=11) ticks = DataFrame([{'buy': 1, 'date': oldtime}]) @@ -151,15 +156,16 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): return_value=DataFrame(ticks) ) ) - assert (False, False) == _ANALYZE.get_signal('xyz', default_conf['ticker_interval']) + assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) assert log_has( 'Outdated history for pair xyz. Last tick is 11 minutes old', caplog.record_tuples ) -def test_get_signal_handles_exceptions(mocker): - mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) +def test_get_signal_handles_exceptions(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) mocker.patch.multiple( 'freqtrade.analyze.Analyze', analyze_ticker=MagicMock( @@ -167,7 +173,7 @@ def test_get_signal_handles_exceptions(mocker): ) ) - assert _ANALYZE.get_signal('ETH/BTC', '5m') == (False, False) + assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) def test_parse_ticker_dataframe(ticker_history): From 75d02df60dc6fbb347d30eaee63ec82427d399ec Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 21:10:42 +0200 Subject: [PATCH 084/239] add exchange to call get_singal --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5f69b8115..e708ba57f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -426,7 +426,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ (buy, sell) = (False, False) if self.config.get('experimental', {}).get('use_sell_signal'): - (buy, sell) = self.analyze.get_signal(trade.pair, self.analyze.get_ticker_interval()) + (buy, sell) = self.analyze.get_signal(self.exchange, trade.pair, self.analyze.get_ticker_interval()) if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) From 975b42caa37e320c6b602d7ee4964d8b56ec5dc8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 21:11:10 +0200 Subject: [PATCH 085/239] fix tests for exchange objectify --- freqtrade/tests/test_freqtradebot.py | 133 ++++++++++++++------------- 1 file changed, 69 insertions(+), 64 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1d272428e..797456328 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -32,7 +32,8 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) patch_coinmarketcap(mocker) return FreqtradeBot(config) @@ -47,7 +48,7 @@ def patch_get_signal(mocker, value=(True, False)) -> None: """ mocker.patch( 'freqtrade.freqtradebot.Analyze.get_signal', - side_effect=lambda s, t: value + side_effect=lambda e, s, t: value ) @@ -187,9 +188,9 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: Test _gen_pair_whitelist() method """ freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.freqtradebot.exchange.get_tickers', tickers) - mocker.patch('freqtrade.freqtradebot.exchange.exchange_has', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + # mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) # Test to retrieved BTC sorted on quoteVolume (default) whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC') @@ -224,7 +225,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -261,7 +262,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=buy_mock, @@ -285,7 +286,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -306,7 +307,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocke patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -333,7 +334,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -362,7 +363,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker_history=MagicMock(return_value=20), get_balance=MagicMock(return_value=20), @@ -387,7 +388,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, @@ -428,7 +429,7 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, @@ -450,7 +451,7 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> msg_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, @@ -474,7 +475,7 @@ def test_process_trade_handling( patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, @@ -495,29 +496,32 @@ def test_process_trade_handling( assert result is False -def test_balance_fully_ask_side(mocker) -> None: +def test_balance_fully_ask_side(mocker, default_conf) -> None: """ Test get_target_bid() method """ - freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 0.0}}) + default_conf['bid_strategy']['ask_last_balance'] = 0.0 + freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20 -def test_balance_fully_last_side(mocker) -> None: +def test_balance_fully_last_side(mocker, default_conf) -> None: """ Test get_target_bid() method """ - freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 1.0}}) + default_conf['bid_strategy']['ask_last_balance'] = 1.0 + freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10 -def test_balance_bigger_last_ask(mocker) -> None: +def test_balance_bigger_last_ask(mocker, default_conf) -> None: """ Test get_target_bid() method """ - freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 1.0}}) + default_conf['bid_strategy']['ask_last_balance'] = 1.0 + freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5 @@ -556,8 +560,8 @@ def test_process_maybe_execute_sell(mocker, default_conf, limit_buy_order, caplo freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) - mocker.patch('freqtrade.freqtradebot.exchange.get_order', return_value=limit_buy_order) - mocker.patch('freqtrade.freqtradebot.exchange.get_trades_for_order', return_value=[]) + mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=limit_buy_order['amount']) @@ -590,7 +594,7 @@ def test_process_maybe_execute_sell_exception(mocker, default_conf, Test the exceptions in process_maybe_execute_sell() """ freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.freqtradebot.exchange.get_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order) trade = MagicMock() trade.open_order_id = '123' @@ -620,7 +624,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock patch_get_signal(mocker) patch_RPCManager(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00001172, @@ -669,7 +673,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -727,7 +731,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, ca patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -764,7 +768,7 @@ def test_handle_trade_experimental( patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -794,7 +798,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fe patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -824,7 +828,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe cancel_order_mock = MagicMock() patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old), @@ -865,7 +869,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(return_value=limit_sell_order_old), @@ -905,7 +909,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), @@ -953,7 +957,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - handle_timedout_limit_sell=MagicMock(), ) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), @@ -993,7 +997,7 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None: patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), cancel_order=cancel_order_mock ) @@ -1019,7 +1023,7 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: cancel_order_mock = MagicMock() patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), cancel_order=cancel_order_mock ) @@ -1045,7 +1049,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -1061,7 +1065,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N # Increase the price and sell it mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) @@ -1087,7 +1091,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -1102,7 +1106,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) # Decrease the price and sell it mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_down ) @@ -1127,7 +1131,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -1142,7 +1146,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, # Increase the price and sell it mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) @@ -1168,7 +1172,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -1183,7 +1187,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, # Decrease the price and sell it mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_down ) @@ -1207,7 +1211,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, mock patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00002172, @@ -1240,7 +1244,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, moc patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00002172, @@ -1273,7 +1277,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00000172, @@ -1306,7 +1310,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke patch_coinmarketcap(mocker) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00000172, @@ -1337,12 +1341,12 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca Test get_real_amount - fee in quote currency """ - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -1364,12 +1368,12 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): Test get_real_amount - fee in quote currency """ - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=[]) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) amount = buy_order_fee['amount'] trade = Trade( pair='LTC/ETH', @@ -1395,8 +1399,8 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -1421,8 +1425,8 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -1444,8 +1448,8 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order2) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) amount = float(sum(x['amount'] for x in trades_for_order2)) trade = Trade( pair='LTC/ETH', @@ -1472,8 +1476,9 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=[trades_for_order]) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', + return_value=[trades_for_order]) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( pair='LTC/ETH', @@ -1500,8 +1505,8 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=[]) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( pair='LTC/ETH', @@ -1525,8 +1530,8 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -1547,7 +1552,7 @@ def test_get_real_amount_open_trade(default_conf, mocker): patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) amount = 12345 trade = Trade( pair='LTC/ETH', From 63b568989aeb7abb8fdbc5201586113f8134fa87 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 21:24:36 +0200 Subject: [PATCH 086/239] Fix rpc for exchange objectify --- freqtrade/rpc/rpc.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 34802f920..d58c265a7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -12,7 +12,7 @@ import sqlalchemy as sql from numpy import mean, nan_to_num from pandas import DataFrame -from freqtrade import exchange +from freqtrade.exchange import Exchange from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State @@ -71,9 +71,9 @@ class RPC(object): for trade in trades: order = None if trade.open_order_id: - order = exchange.get_order(trade.open_order_id, trade.pair) + order = self._freqtrade.exchange.get_order(trade.open_order_id, trade.pair) # calculate profit and send message to user - current_rate = exchange.get_ticker(trade.pair, False)['bid'] + current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] current_profit = trade.calc_profit_percent(current_rate) fmt_close_profit = '{:.2f}%'.format( round(trade.close_profit * 100, 2) @@ -91,7 +91,7 @@ class RPC(object): .format( trade_id=trade.id, pair=trade.pair, - market_url=exchange.get_pair_detail_url(trade.pair), + market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), date=arrow.get(trade.open_date).humanize(), open_rate=trade.open_rate, close_rate=trade.close_rate, @@ -116,7 +116,7 @@ class RPC(object): trades_list = [] for trade in trades: # calculate profit and send message to user - current_rate = exchange.get_ticker(trade.pair, False)['bid'] + current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] trades_list.append([ trade.id, trade.pair, @@ -201,7 +201,7 @@ class RPC(object): profit_closed_percent.append(profit_percent) else: # Get current rate - current_rate = exchange.get_ticker(trade.pair, False)['bid'] + current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] profit_percent = trade.calc_profit_percent(rate=current_rate) profit_all_coin.append( @@ -258,7 +258,7 @@ class RPC(object): """ Returns current account balance per crypto """ output = [] total = 0.0 - for coin, balance in exchange.get_balances().items(): + for coin, balance in self._freqtrade.exchange.get_balances().items(): if not balance['total']: continue @@ -266,9 +266,9 @@ class RPC(object): rate = 1.0 else: if coin == 'USDT': - rate = 1.0 / exchange.get_ticker('BTC/USDT', False)['bid'] + rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid'] else: - rate = exchange.get_ticker(coin + '/BTC', False)['bid'] + rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] est_btc: float = rate * balance['total'] total = total + est_btc output.append( @@ -318,13 +318,13 @@ class RPC(object): def _exec_forcesell(trade: Trade) -> None: # Check if there is there is an open order if trade.open_order_id: - order = exchange.get_order(trade.open_order_id, trade.pair) + order = self._freqtrade.exchange.get_order(trade.open_order_id, trade.pair) # Cancel open LIMIT_BUY orders and close trade if order and order['status'] == 'open' \ and order['type'] == 'limit' \ and order['side'] == 'buy': - exchange.cancel_order(trade.open_order_id, trade.pair) + self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair) trade.close(order.get('price') or trade.open_rate) # Do the best effort, if we don't know 'filled' amount, don't try selling if order['filled'] is None: @@ -338,7 +338,7 @@ class RPC(object): return # Get current rate and execute sell - current_rate = exchange.get_ticker(trade.pair, False)['bid'] + current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] self._freqtrade.execute_sell(trade, current_rate) # ---- EOF def _exec_forcesell ---- From 64e09f74a1d271ba93274ecf1edb048073764b17 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 21:24:51 +0200 Subject: [PATCH 087/239] fix rpc tests --- freqtrade/tests/rpc/test_rpc.py | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index b49b7fdcb..cc3a78a0e 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -33,7 +33,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -79,7 +79,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -112,7 +112,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -167,7 +167,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -190,7 +190,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) @@ -205,7 +205,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) @@ -243,7 +243,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -262,7 +262,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, trade.update(limit_buy_order) # Update the ticker with a market going up mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_up, get_fee=fee @@ -314,7 +314,7 @@ def test_rpc_balance_handle(default_conf, mocker): mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_balances=MagicMock(return_value=mock_balance) ) @@ -342,7 +342,7 @@ def test_rpc_start(mocker, default_conf) -> None: patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock() ) @@ -368,7 +368,7 @@ def test_rpc_stop(mocker, default_conf) -> None: patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock() ) @@ -396,7 +396,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: cancel_order_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, cancel_order=cancel_order_mock, @@ -441,7 +441,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: trade = Trade.query.filter(Trade.id == '1').first() filled_amount = trade.amount / 2 mocker.patch( - 'freqtrade.freqtradebot.exchange.get_order', + 'freqtrade.exchange.Exchange.get_order', return_value={ 'status': 'open', 'type': 'limit', @@ -460,7 +460,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: amount = trade.amount # make an limit-buy open trade, if there is no 'filled', don't sell it mocker.patch( - 'freqtrade.freqtradebot.exchange.get_order', + 'freqtrade.exchange.Exchange.get_order', return_value={ 'status': 'open', 'type': 'limit', @@ -476,7 +476,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.create_trade() # make an limit-sell open trade mocker.patch( - 'freqtrade.freqtradebot.exchange.get_order', + 'freqtrade.exchange.Exchange.get_order', return_value={ 'status': 'open', 'type': 'limit', @@ -497,7 +497,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_balances=MagicMock(return_value=ticker), get_ticker=ticker, @@ -535,7 +535,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_balances=MagicMock(return_value=ticker), get_ticker=ticker, From c83e8b7cb585bc7fafc97407b4c1ebc56f264073 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 21:46:56 +0200 Subject: [PATCH 088/239] fix rpc_test --- freqtrade/tests/conftest.py | 9 ++- freqtrade/tests/rpc/test_rpc_telegram.py | 93 ++++++++++-------------- 2 files changed, 43 insertions(+), 59 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index ed900e6bb..ce22cd193 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -27,13 +27,16 @@ def log_has(line, logs): False) -def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: - +def patch_exchange(mocker, api_mock=None) -> None: mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) else: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) + + +def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: + patch_exchange(mocker, api_mock) exchange = Exchange(config) return exchange @@ -51,7 +54,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + patch_exchange(mocker, None) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index f022c09e4..1b07a2143 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -20,7 +20,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc.telegram import Telegram from freqtrade.rpc.telegram import authorized_only from freqtrade.state import State -from freqtrade.tests.conftest import get_patched_freqtradebot, log_has +from freqtrade.tests.conftest import get_patched_freqtradebot,patch_exchange, log_has from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap @@ -100,7 +100,7 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + patch_exchange(mocker, None) chat = Chat(0, 0) update = Update(randint(1, 100)) @@ -131,8 +131,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) - + patch_exchange(mocker, None) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) @@ -162,7 +161,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + patch_exchange(mocker) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), Chat(0, 0)) @@ -198,7 +197,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_pair_detail_url=MagicMock(), @@ -238,7 +237,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, @@ -284,7 +283,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), @@ -341,7 +340,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, return_value=15000.0 ) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -410,7 +409,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker ) @@ -450,7 +449,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -484,7 +483,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, msg_mock.reset_mock() # Update the ticker with a market going up - mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', ticker_sell_up) + mocker.patch('freqtrade.exchange.Exchange.get_ticker', ticker_sell_up) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -549,9 +548,8 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) - mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=mock_balance) - mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', side_effect=mock_ticker) + mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=mock_balance) + mocker.patch('freqtrade.exchange.Exchange.get_ticker', side_effect=mock_ticker) msg_mock = MagicMock() mocker.patch.multiple( @@ -560,7 +558,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: _send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -579,9 +577,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: Test _balance() method when the Exchange platform returns nothing """ patch_get_signal(mocker, (True, False)) - patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) - mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value={}) + mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) msg_mock = MagicMock() mocker.patch.multiple( @@ -590,7 +586,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: _send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -603,17 +599,14 @@ def test_start_handle(default_conf, update, mocker) -> None: """ Test _start() method """ - patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -627,17 +620,14 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: """ Test _start() method """ - patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.RUNNING @@ -653,16 +643,14 @@ def test_stop_handle(default_conf, update, mocker) -> None: Test _stop() method """ patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.RUNNING @@ -678,16 +666,14 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: Test _stop() method """ patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -701,16 +687,14 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: def test_reload_conf_handle(default_conf, update, mocker) -> None: """ Test _reload_conf() method """ patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.RUNNING @@ -731,7 +715,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, moc rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -747,7 +731,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, moc assert trade # Increase the price and sell it - mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', ticker_sell_up) + mocker.patch('freqtrade.exchange.Exchange.get_ticker', ticker_sell_up) update.message.text = '/forcesell 1' telegram._forcesell(bot=MagicMock(), update=update) @@ -771,7 +755,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_do rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -785,7 +769,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_do # Decrease the price and sell it mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_down ) @@ -814,9 +798,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - mocker.patch('freqtrade.exchange.get_pair_detail_url', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.get_pair_detail_url', MagicMock()) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -853,7 +837,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) @@ -896,7 +880,7 @@ def test_performance_handle(default_conf, update, ticker, fee, _send_msg=msg_mock ) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee @@ -936,7 +920,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) @@ -960,12 +944,12 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: _send_msg=msg_mock ) mocker.patch.multiple( - 'freqtrade.freqtradebot.exchange', + 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}) ) - mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) @@ -995,14 +979,14 @@ def test_help_handle(default_conf, update, mocker) -> None: Test _help() method """ patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) telegram._help(bot=MagicMock(), update=update) @@ -1015,14 +999,13 @@ def test_version_handle(default_conf, update, mocker) -> None: Test _version() method """ patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram._version(bot=MagicMock(), update=update) @@ -1035,11 +1018,10 @@ def test_send_msg(default_conf, mocker) -> None: Test send_msg() method """ patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) bot = MagicMock() - freqtradebot = FreqtradeBot(conf) + freqtradebot = get_patched_freqtradebot(mocker, conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True @@ -1052,12 +1034,11 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None: Test send_msg() method """ patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) bot = MagicMock() bot.send_message = MagicMock(side_effect=NetworkError('Oh snap')) - freqtradebot = FreqtradeBot(conf) + freqtradebot = get_patched_freqtradebot(mocker, conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True From 251f7db3ca1ec7ef123a36eefd070bc8aeead60c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 22:11:32 +0200 Subject: [PATCH 089/239] require exchange object to delete pairs --- freqtrade/optimize/__init__.py | 11 ++++++++--- freqtrade/optimize/backtesting.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 3e2fb4554..1e76808e7 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -7,7 +7,7 @@ import os from typing import Optional, List, Dict, Tuple, Any import arrow -from freqtrade import misc, constants +from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange @@ -83,6 +83,7 @@ def load_data(datadir: str, ticker_interval: str, pairs: List[str], refresh_pairs: Optional[bool] = False, + exchange: Optional[Exchange] = None, timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]: """ Loads ticker history data for the given parameters @@ -93,7 +94,10 @@ def load_data(datadir: str, # If the user force the refresh of pairs if refresh_pairs: logger.info('Download data for all pairs and store them in %s', datadir) - download_pairs(datadir, pairs, ticker_interval, timerange=timerange) + if not exchange: + raise OperationalException("Exchange needs to be initialized when " + "calling load_data with refresh_pairs=True") + download_pairs(datadir, exchange, pairs, ticker_interval, timerange=timerange) for pair in pairs: pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) @@ -119,13 +123,14 @@ def make_testdata_path(datadir: str) -> str: ) -def download_pairs(datadir, pairs: List[str], +def download_pairs(datadir, exchange: Exchange, pairs: List[str], ticker_interval: str, timerange: TimeRange = TimeRange(None, None, 0, 0)) -> bool: """For each pairs passed in parameters, download the ticker intervals""" for pair in pairs: try: download_backtesting_testdata(datadir, + exchange=exchange, pair=pair, tick_interval=ticker_interval, timerange=timerange) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 58b552237..9a19d1412 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -267,6 +267,7 @@ class Backtesting(object): pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, timerange=timerange ) From 52d36c33cf6e905753ef411741727d0f03a10691 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 22:12:22 +0200 Subject: [PATCH 090/239] fix optimie test --- freqtrade/tests/optimize/test_optimize.py | 60 +++++++++++++---------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index bac8a6b36..624f8ed77 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -12,7 +12,7 @@ from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \ load_cached_data_for_updating from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has +from freqtrade.tests.conftest import log_has, get_patched_exchange # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -49,12 +49,11 @@ def _clean_test_file(file: str) -> None: os.rename(file_swp, file) -def test_load_data_30min_ticker(ticker_history, mocker, caplog) -> None: +def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: """ Test load_data() with 30 min ticker """ - mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') @@ -63,11 +62,11 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog) -> None: _clean_test_file(file) -def test_load_data_5min_ticker(ticker_history, mocker, caplog) -> None: +def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: """ Test load_data() with 5 min ticker """ - mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) @@ -81,7 +80,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) @@ -91,12 +90,12 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: _clean_test_file(file) -def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: +def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_conf) -> None: """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') _backup_file(file) @@ -114,6 +113,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: optimize.load_data(None, ticker_interval='1m', refresh_pairs=True, + exchange=exchange, pairs=['MEME/BTC']) assert os.path.isfile(file) is True assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) @@ -124,9 +124,9 @@ def test_testdata_path() -> None: assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) -def test_download_pairs(ticker_history, mocker) -> None: - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - +def test_download_pairs(ticker_history, mocker, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json') @@ -140,7 +140,8 @@ def test_download_pairs(ticker_history, mocker) -> None: assert os.path.isfile(file1_1) is False assert os.path.isfile(file2_1) is False - assert download_pairs(None, pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='1m') is True + assert download_pairs(None, exchange, + pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='1m') is True assert os.path.isfile(file1_1) is True assert os.path.isfile(file2_1) is True @@ -152,7 +153,8 @@ def test_download_pairs(ticker_history, mocker) -> None: assert os.path.isfile(file1_5) is False assert os.path.isfile(file2_5) is False - assert download_pairs(None, pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='5m') is True + assert download_pairs(None, exchange, + pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='5m') is True assert os.path.isfile(file1_5) is True assert os.path.isfile(file2_5) is True @@ -265,30 +267,32 @@ def test_load_cached_data_for_updating(mocker) -> None: assert start_ts is None -def test_download_pairs_exception(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) +def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) + exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') _backup_file(file1_1) _backup_file(file1_5) - download_pairs(None, pairs=['MEME/BTC'], ticker_interval='1m') + download_pairs(None, exchange, pairs=['MEME/BTC'], ticker_interval='1m') # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) -def test_download_backtesting_testdata(ticker_history, mocker) -> None: - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) +def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + exchange = get_patched_exchange(mocker, default_conf) # Download a 1 min ticker file file1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'XEL_BTC-1m.json') _backup_file(file1) - download_backtesting_testdata(None, pair="XEL/BTC", tick_interval='1m') + download_backtesting_testdata(None, exchange, pair="XEL/BTC", tick_interval='1m') assert os.path.isfile(file1) is True _clean_test_file(file1) @@ -296,21 +300,21 @@ def test_download_backtesting_testdata(ticker_history, mocker) -> None: file2 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'STORJ_BTC-5m.json') _backup_file(file2) - download_backtesting_testdata(None, pair="STORJ/BTC", tick_interval='5m') + download_backtesting_testdata(None, exchange, pair="STORJ/BTC", tick_interval='5m') assert os.path.isfile(file2) is True _clean_test_file(file2) -def test_download_backtesting_testdata2(mocker) -> None: +def test_download_backtesting_testdata2(mocker, default_conf) -> None: tick = [ [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) - - download_backtesting_testdata(None, pair="UNITTEST/BTC", tick_interval='1m') - download_backtesting_testdata(None, pair="UNITTEST/BTC", tick_interval='3m') + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=tick) + exchange = get_patched_exchange(mocker, default_conf) + download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') + download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') assert json_dump_mock.call_count == 2 @@ -326,8 +330,10 @@ def test_load_tickerdata_file() -> None: def test_init(default_conf, mocker) -> None: + exchange = get_patched_exchange(mocker, default_conf) assert {} == optimize.load_data( '', + exchange=exchange, pairs=[], refresh_pairs=True, ticker_interval=default_conf['ticker_interval'] From ace519847536f53980be2eba89ed1bcc944031a6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 22:29:57 +0200 Subject: [PATCH 091/239] fix optimize tests --- freqtrade/tests/optimize/test_backtesting.py | 58 ++++++++++---------- freqtrade/tests/optimize/test_hyperopt.py | 15 +++-- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 31080c503..58fa3a3f3 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -83,7 +83,7 @@ def load_data_test(what): def simple_backtest(config, contour, num_results, mocker) -> None: - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(config) data = load_data_test(contour) @@ -101,7 +101,8 @@ def simple_backtest(config, contour, num_results, mocker) -> None: assert len(results) == num_results -def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, timerange=None): +def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, + timerange=None, exchange=None): tickerdata = optimize.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': tickerdata} return pairdata @@ -118,7 +119,7 @@ def _load_pair_as_ticks(pair, tickfreq): def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) data = trim_dictlist(data, -201) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(conf) return { 'stake_amount': conf['stake_amount'], @@ -272,8 +273,8 @@ def test_start(mocker, fee, default_conf, caplog) -> None: Test start() function """ start_mock = MagicMock() - mocker.patch('freqtrade.exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.optimize.backtesting.Backtesting.start', start_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) @@ -296,7 +297,7 @@ def test_backtesting_init(mocker, default_conf) -> None: """ Test Backtesting._init() method """ - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert isinstance(backtesting.analyze, Analyze) @@ -310,7 +311,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: """ Test Backtesting.tickerdata_to_dataframe() method """ - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) timerange = TimeRange(None, 'line', 0, -100) tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} @@ -329,7 +330,7 @@ def test_get_timeframe(default_conf, mocker) -> None: """ Test Backtesting.get_timeframe() method """ - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(default_conf) data = backtesting.tickerdata_to_dataframe( @@ -348,7 +349,7 @@ def test_generate_text_table(default_conf, mocker): """ Test Backtesting.generate_text_table() method """ - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(default_conf) results = pd.DataFrame( @@ -385,8 +386,8 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.get_ticker_history') - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), @@ -426,8 +427,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.get_ticker_history') - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), @@ -454,8 +455,8 @@ def test_backtest(default_conf, fee, mocker) -> None: """ Test Backtesting.backtest() method """ - mocker.patch('freqtrade.exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(default_conf) data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) @@ -476,8 +477,8 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: """ Test Backtesting.backtest() method with 1 min ticker """ - mocker.patch('freqtrade.exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(default_conf) # Run a backtesting for an exiting 5min ticker_interval @@ -499,7 +500,7 @@ def test_processed(default_conf, mocker) -> None: """ Test Backtesting.backtest() method with offline data """ - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(default_conf) dict_of_tickerrows = load_data_test('raise') @@ -513,7 +514,7 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: - mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) tests = [['raise', 18], ['lower', 0], ['sine', 16]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres, mocker) @@ -521,8 +522,8 @@ def test_backtest_pricecontours(default_conf, fee, mocker) -> None: # Test backtest using offline data (testdata directory) def test_backtest_ticks(default_conf, fee, mocker): - mocker.patch('freqtrade.exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) ticks = [1, 5] fun = Backtesting(default_conf).populate_buy_trend for _ in ticks: @@ -541,7 +542,6 @@ def test_backtest_clash_buy_sell(mocker, default_conf): sell_value = 1 return _trend(dataframe, buy_value, sell_value) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) backtest_conf = _make_backtest_conf(mocker, conf=default_conf) backtesting = Backtesting(default_conf) backtesting.populate_buy_trend = fun # Override @@ -557,7 +557,6 @@ def test_backtest_only_sell(mocker, default_conf): sell_value = 1 return _trend(dataframe, buy_value, sell_value) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) backtest_conf = _make_backtest_conf(mocker, conf=default_conf) backtesting = Backtesting(default_conf) backtesting.populate_buy_trend = fun # Override @@ -567,8 +566,7 @@ def test_backtest_only_sell(mocker, default_conf): def test_backtest_alternate_buy_sell(default_conf, fee, mocker): - mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC') backtesting = Backtesting(default_conf) backtesting.populate_buy_trend = _trend_alternate # Override @@ -583,8 +581,8 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): def test_backtest_record(default_conf, fee, mocker): names = [] records = [] - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch( 'freqtrade.optimize.backtesting.file_dump_json', new=lambda n, r: (names.append(n), records.append(r)) @@ -632,9 +630,9 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_live(default_conf, mocker, caplog): conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.get_ticker_history', - new=lambda n, i: _load_pair_as_ticks(n, i)) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', + new=lambda s, n, i: _load_pair_as_ticks(n, i)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) mocker.patch('freqtrade.configuration.open', mocker.mock_open( diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9b66a29e2..616d56c8f 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -22,8 +22,7 @@ _HYPEROPT = None def init_hyperopt(default_conf, mocker): global _HYPEROPT_INITIALIZED, _HYPEROPT if not _HYPEROPT_INITIALIZED: - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) _HYPEROPT = Hyperopt(default_conf) _HYPEROPT_INITIALIZED = True @@ -66,7 +65,7 @@ def test_start(mocker, default_conf, caplog) -> None: lambda *args, **kwargs: default_conf ) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) args = [ '--config', 'config.json', @@ -182,7 +181,7 @@ def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) @@ -226,7 +225,7 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> conf.update({'epochs': 1}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) @@ -267,7 +266,7 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) @@ -338,7 +337,7 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: trials = create_trials(mocker) mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) conf = deepcopy(default_conf) @@ -503,7 +502,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: 'freqtrade.optimize.hyperopt.Hyperopt.backtest', MagicMock(return_value=backtest_result) ) - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) optimizer_param = { 'adx': {'enabled': False}, From e194af8d254d599f662e069220cbecb092c1a3fc Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 22:32:56 +0200 Subject: [PATCH 092/239] Streamline validate_pair patching --- freqtrade/tests/optimize/test_backtesting.py | 32 ++++++++++---------- freqtrade/tests/optimize/test_hyperopt.py | 18 ++++++----- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 58fa3a3f3..1702818b1 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -15,7 +15,7 @@ from freqtrade import optimize from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration -from freqtrade.tests.conftest import log_has +from freqtrade.tests.conftest import log_has, patch_exchange def get_args(args) -> List[str]: @@ -83,7 +83,7 @@ def load_data_test(what): def simple_backtest(config, contour, num_results, mocker) -> None: - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(config) data = load_data_test(contour) @@ -119,7 +119,7 @@ def _load_pair_as_ticks(pair, tickfreq): def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) data = trim_dictlist(data, -201) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(conf) return { 'stake_amount': conf['stake_amount'], @@ -274,7 +274,7 @@ def test_start(mocker, fee, default_conf, caplog) -> None: """ start_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.start', start_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) @@ -297,7 +297,7 @@ def test_backtesting_init(mocker, default_conf) -> None: """ Test Backtesting._init() method """ - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert isinstance(backtesting.analyze, Analyze) @@ -311,7 +311,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: """ Test Backtesting.tickerdata_to_dataframe() method """ - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) timerange = TimeRange(None, 'line', 0, -100) tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} @@ -330,7 +330,7 @@ def test_get_timeframe(default_conf, mocker) -> None: """ Test Backtesting.get_timeframe() method """ - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(default_conf) data = backtesting.tickerdata_to_dataframe( @@ -349,7 +349,7 @@ def test_generate_text_table(default_conf, mocker): """ Test Backtesting.generate_text_table() method """ - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(default_conf) results = pd.DataFrame( @@ -387,7 +387,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), @@ -428,7 +428,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), @@ -456,7 +456,7 @@ def test_backtest(default_conf, fee, mocker) -> None: Test Backtesting.backtest() method """ mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(default_conf) data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) @@ -478,7 +478,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: Test Backtesting.backtest() method with 1 min ticker """ mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(default_conf) # Run a backtesting for an exiting 5min ticker_interval @@ -500,7 +500,7 @@ def test_processed(default_conf, mocker) -> None: """ Test Backtesting.backtest() method with offline data """ - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) backtesting = Backtesting(default_conf) dict_of_tickerrows = load_data_test('raise') @@ -523,7 +523,7 @@ def test_backtest_pricecontours(default_conf, fee, mocker) -> None: # Test backtest using offline data (testdata directory) def test_backtest_ticks(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) ticks = [1, 5] fun = Backtesting(default_conf).populate_buy_trend for _ in ticks: @@ -581,7 +581,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): def test_backtest_record(default_conf, fee, mocker): names = [] records = [] - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch( 'freqtrade.optimize.backtesting.file_dump_json', @@ -632,7 +632,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) mocker.patch('freqtrade.configuration.open', mocker.mock_open( diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 616d56c8f..7de6bcb7c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -10,7 +10,7 @@ import pytest from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start from freqtrade.strategy.resolver import StrategyResolver -from freqtrade.tests.conftest import log_has +from freqtrade.tests.conftest import log_has, patch_exchange from freqtrade.tests.optimize.test_backtesting import get_args # Avoid to reinit the same object again and again @@ -22,7 +22,7 @@ _HYPEROPT = None def init_hyperopt(default_conf, mocker): global _HYPEROPT_INITIALIZED, _HYPEROPT if not _HYPEROPT_INITIALIZED: - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) _HYPEROPT = Hyperopt(default_conf) _HYPEROPT_INITIALIZED = True @@ -65,7 +65,8 @@ def test_start(mocker, default_conf, caplog) -> None: lambda *args, **kwargs: default_conf ) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) + args = [ '--config', 'config.json', @@ -181,7 +182,8 @@ def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) + StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) @@ -225,7 +227,7 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> conf.update({'epochs': 1}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) @@ -266,7 +268,7 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) @@ -337,7 +339,7 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: trials = create_trials(mocker) mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) conf = deepcopy(default_conf) @@ -502,7 +504,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: 'freqtrade.optimize.hyperopt.Hyperopt.backtest', MagicMock(return_value=backtest_result) ) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) optimizer_param = { 'adx': {'enabled': False}, From 6e6ec969ebe72f60929846d4e2f25e0ea5d08b56 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 22:38:31 +0200 Subject: [PATCH 093/239] cleanup mockings --- freqtrade/tests/test_freqtradebot.py | 5 ++--- freqtrade/tests/test_main.py | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 797456328..9eae2fdf5 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade import DependencyException, OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.state import State -from freqtrade.tests.conftest import log_has, patch_coinmarketcap +from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange # Functions for recurrent object patching @@ -32,8 +32,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) patch_coinmarketcap(mocker) return FreqtradeBot(config) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 9640a7350..414dcdbe9 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -13,7 +13,7 @@ from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main, set_loggers, reconfigure from freqtrade.state import State -from freqtrade.tests.conftest import log_has +from freqtrade.tests.conftest import log_has, patch_exchange def test_parse_args_backtesting(mocker) -> None: @@ -70,6 +70,7 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None: Test main() function In this test we are skipping the while True loop by throwing an exception. """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', _init_modules=MagicMock(), @@ -97,6 +98,7 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: Test main() function In this test we are skipping the while True loop by throwing an exception. """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', _init_modules=MagicMock(), @@ -124,6 +126,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: Test main() function In this test we are skipping the while True loop by throwing an exception. """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', _init_modules=MagicMock(), @@ -151,6 +154,7 @@ def test_main_reload_conf(mocker, default_conf, caplog) -> None: Test main() function In this test we are skipping the while True loop by throwing an exception. """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', _init_modules=MagicMock(), @@ -178,6 +182,7 @@ def test_main_reload_conf(mocker, default_conf, caplog) -> None: def test_reconfigure(mocker, default_conf) -> None: """ Test recreate() function """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', _init_modules=MagicMock(), From 2b099a89e430f3e78ad5b7fa6f7cc0f2bbf9d40c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 22:42:28 +0200 Subject: [PATCH 094/239] fix styling issues --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/rpc/rpc.py | 1 - freqtrade/tests/optimize/test_hyperopt.py | 2 -- freqtrade/tests/rpc/test_rpc_telegram.py | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e708ba57f..dd35700a4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -54,7 +54,7 @@ class FreqtradeBot(object): self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None - self.exchange = None + self.exchange = Exchange(self.config) self._init_modules() @@ -66,7 +66,6 @@ class FreqtradeBot(object): # Initialize all modules persistence.init(self.config) - self.exchange = Exchange(self.config) # Set initial application state initial_state = self.config.get('initial_state') @@ -426,7 +425,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ (buy, sell) = (False, False) if self.config.get('experimental', {}).get('use_sell_signal'): - (buy, sell) = self.analyze.get_signal(self.exchange, trade.pair, self.analyze.get_ticker_interval()) + (buy, sell) = self.analyze.get_signal(self.exchange, + trade.pair, self.analyze.get_ticker_interval()) if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index d58c265a7..ee6ecb770 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -12,7 +12,6 @@ import sqlalchemy as sql from numpy import mean, nan_to_num from pandas import DataFrame -from freqtrade.exchange import Exchange from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 7de6bcb7c..8ad1932af 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -67,7 +67,6 @@ def test_start(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) patch_exchange(mocker) - args = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', @@ -184,7 +183,6 @@ def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) patch_exchange(mocker) - StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 1b07a2143..0a7d9690f 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -20,7 +20,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc.telegram import Telegram from freqtrade.rpc.telegram import authorized_only from freqtrade.state import State -from freqtrade.tests.conftest import get_patched_freqtradebot,patch_exchange, log_has +from freqtrade.tests.conftest import get_patched_freqtradebot, patch_exchange, log_has from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap From d156de39f1ee7fb2416508328e90cc60a9479fbe Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 22:57:49 +0200 Subject: [PATCH 095/239] Increase test-coverage --- freqtrade/tests/exchange/test_exchange.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index c66116ae0..bb2cdc6e0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -585,6 +585,10 @@ def test_get_fee(default_conf, mocker): def test_get_amount_lots(default_conf, mocker): api_mock = MagicMock() api_mock.amount_to_lots = MagicMock(return_value=1.0) + api_mock.markets = None + marketmock = MagicMock() + api_mock.load_markets = marketmock exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.get_amount_lots('LTC/BTC', 1.54) == 1 + assert marketmock.call_count == 1 From c9f8dfc6c55198d93a165721bdf1601ebae6e1b3 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 23:05:44 +0200 Subject: [PATCH 096/239] increase get_fee coverage --- freqtrade/tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index bb2cdc6e0..5541c63c6 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -581,6 +581,20 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 + # test Exceptions + api_mock = MagicMock() + api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + with pytest.raises(OperationalException): + exchange.get_fee() + + api_mock = MagicMock() + api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + with pytest.raises(TemporaryError): + exchange.get_fee() + assert api_mock.calculate_fee.call_count == API_RETRY_COUNT + 1 + def test_get_amount_lots(default_conf, mocker): api_mock = MagicMock() From 1e3d722bc2998da849f20a2fe84c9e9fe26c8371 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 23:22:28 +0200 Subject: [PATCH 097/239] add test for get_trades --- freqtrade/tests/exchange/test_exchange.py | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 5541c63c6..4d442d534 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -3,6 +3,7 @@ import logging from copy import deepcopy from random import randint +from datetime import datetime from unittest.mock import MagicMock, PropertyMock import ccxt @@ -569,6 +570,54 @@ def test_get_pair_detail_url(default_conf, mocker): assert 'BTC' in url +def test_get_trades_for_order(default_conf, mocker): + order_id = 'ABCD-ABCD' + since = datetime(2018, 5, 5) + default_conf["dry_run"] = False + mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True) + api_mock = MagicMock() + + api_mock.fetch_my_trades = MagicMock(return_value=[{'id': 'TTR67E-3PFBD-76IISV', + 'order': 'ABCD-ABCD', + 'info': {'pair': 'XLTCZBTC', + 'time': 1519860024.4388, + 'type': 'buy', + 'ordertype': 'limit', + 'price': '20.00000', + 'cost': '38.62000', + 'fee': '0.06179', + 'vol': '5', + 'id': 'ABCD-ABCD'}, + 'timestamp': 1519860024438, + 'datetime': '2018-02-28T23:20:24.438Z', + 'symbol': 'LTC/BTC', + 'type': 'limit', + 'side': 'buy', + 'price': 165.0, + 'amount': 0.2340606, + 'fee': {'cost': 0.06179, 'currency': 'BTC'} + }]) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + orders = exchange.get_trades_for_order(order_id, 'LTC/BTC', since) + assert len(orders) == 1 + assert orders[0]['price'] == 165 + + # test Exceptions + api_mock = MagicMock() + api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + with pytest.raises(OperationalException): + exchange.get_trades_for_order(order_id, 'LTC/BTC', since) + + api_mock = MagicMock() + api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + with pytest.raises(TemporaryError): + exchange.get_trades_for_order(order_id, 'LTC/BTC', since) + assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 + + def test_get_fee(default_conf, mocker): api_mock = MagicMock() api_mock.calculate_fee = MagicMock(return_value={ From 520c7feeab12df5a14df3e5862b6f827d86e7c38 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 17 Jun 2018 23:30:48 +0200 Subject: [PATCH 098/239] Add test for fetch_tickers --- freqtrade/tests/exchange/test_exchange.py | 43 +++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4d442d534..0e7457532 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -256,8 +256,47 @@ def test_get_balances_prod(default_conf, mocker): assert api_mock.fetch_balance.call_count == 1 -# This test is somewhat redundant with -# test_exchange_bittrex.py::test_exchange_bittrex_get_ticker +def test_get_tickers(default_conf, mocker): + api_mock = MagicMock() + tick = {'ETH/BTC': { + 'symbol': 'ETH/BTC', + 'bid': 0.5, + 'ask': 1, + 'last': 42, + }, 'BCH/BTC': { + 'symbol': 'BCH/BTC', + 'bid': 0.6, + 'ask': 0.5, + 'last': 41, + } + } + api_mock.fetch_tickers = MagicMock(return_value=tick) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + # retrieve original ticker + tickers = exchange.get_tickers() + + assert 'ETH/BTC' in tickers + assert 'BCH/BTC' in tickers + assert tickers['ETH/BTC']['bid'] == 0.5 + assert tickers['ETH/BTC']['ask'] == 1 + assert tickers['BCH/BTC']['bid'] == 0.6 + assert tickers['BCH/BTC']['ask'] == 0.5 + + with pytest.raises(TemporaryError): # test retrier + api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_tickers() + + with pytest.raises(OperationalException): + api_mock.fetch_tickers = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_tickers() + + api_mock.fetch_tickers = MagicMock(return_value={}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_tickers() + + def test_get_ticker(default_conf, mocker): api_mock = MagicMock() tick = { From 9bc833166781a44b98818234a29f905d8395deb5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 18 Jun 2018 14:23:05 +0200 Subject: [PATCH 099/239] Update ccxt from 1.14.202 to 1.14.211 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d572304d..96f3169c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.202 +ccxt==1.14.211 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 695beecf1429665ceef354c264746e7b041e3b70 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 19:36:36 +0200 Subject: [PATCH 100/239] add test for get_markets --- freqtrade/tests/exchange/test_exchange.py | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 0e7457532..16992b53f 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -657,6 +657,32 @@ def test_get_trades_for_order(default_conf, mocker): assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 +def test_get_markets(default_conf, mocker, markets): + api_mock = MagicMock() + api_mock.fetch_markets = markets + exchange = get_patched_exchange(mocker, default_conf, api_mock) + ret = exchange.get_markets() + assert isinstance(ret, list) + assert len(ret) == 3 + + assert ret[0]["id"] == "ethbtc" + assert ret[0]["symbol"] == "ETH/BTC" + + # test Exceptions + api_mock = MagicMock() + api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + with pytest.raises(OperationalException): + exchange.get_markets() + + api_mock = MagicMock() + api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + with pytest.raises(TemporaryError): + exchange.get_markets() + assert api_mock.fetch_markets.call_count == API_RETRY_COUNT + 1 + + def test_get_fee(default_conf, mocker): api_mock = MagicMock() api_mock.calculate_fee = MagicMock(return_value={ From ae4c4e77bffbdce1f3612ee84c27ff7ef875115d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 19:46:42 +0200 Subject: [PATCH 101/239] standardize exception tests - add one more --- freqtrade/tests/exchange/test_exchange.py | 41 +++++++++++++---------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 16992b53f..d748bdad2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -292,6 +292,11 @@ def test_get_tickers(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_tickers() + with pytest.raises(OperationalException): + api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_tickers() + api_mock.fetch_tickers = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_tickers() @@ -643,16 +648,16 @@ def test_get_trades_for_order(default_conf, mocker): assert orders[0]['price'] == 165 # test Exceptions - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(OperationalException): + api_mock = MagicMock() + api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(TemporaryError): + api_mock = MagicMock() + api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_trades_for_order(order_id, 'LTC/BTC', since) assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 @@ -669,16 +674,16 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["symbol"] == "ETH/BTC" # test Exceptions - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(OperationalException): + api_mock = MagicMock() + api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_markets() - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(TemporaryError): + api_mock = MagicMock() + api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_markets() assert api_mock.fetch_markets.call_count == API_RETRY_COUNT + 1 @@ -696,16 +701,16 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 # test Exceptions - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(OperationalException): + api_mock = MagicMock() + api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_fee() - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(TemporaryError): + api_mock = MagicMock() + api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_fee() assert api_mock.calculate_fee.call_count == API_RETRY_COUNT + 1 From 162f94872990e56f3f5caa8355d36fa1e40e95d9 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 19:56:23 +0200 Subject: [PATCH 102/239] add test for non-configured exchange --- freqtrade/tests/exchange/test_exchange.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d748bdad2..44bf539d0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -587,11 +587,10 @@ def test_get_id(default_conf, mocker): assert exchange.get_id() == 'binance' -def test_get_pair_detail_url(default_conf, mocker): +def test_get_pair_detail_url(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange.validate_pairs', side_effect=lambda s: True) default_conf['exchange']['name'] = 'binance' - # exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange = Exchange(default_conf) url = exchange.get_pair_detail_url('TKN/ETH') @@ -613,6 +612,12 @@ def test_get_pair_detail_url(default_conf, mocker): assert 'LOOONG' in url assert 'BTC' in url + default_conf['exchange']['name'] = 'poloniex' + exchange = Exchange(default_conf) + url = exchange.get_pair_detail_url('LOOONG/BTC') + assert '' == url + assert log_has('Could not get exchange url for Poloniex', caplog.record_tuples) + def test_get_trades_for_order(default_conf, mocker): order_id = 'ABCD-ABCD' From c31519fdb29ae21133343786d7f4769c67b2a454 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 22:07:15 +0200 Subject: [PATCH 103/239] lowercase _api object --- freqtrade/exchange/__init__.py | 55 ++++++++++++++++------------------ 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 0d303fa4a..9f07d66d2 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -43,7 +43,7 @@ def retrier(f): class Exchange(object): # Current selected exchange - _API: ccxt.Exchange = None + _api: ccxt.Exchange = None _CONF: Dict = {} _CACHED_TICKER: Dict[str, Any] = {} @@ -55,18 +55,15 @@ class Exchange(object): Initializes this module with the given config, it does basic validation whether the specified exchange and pairs are valid. - :param config: config to use :return: None """ - self._API - self._CONF.update(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] - self._API = self._init_ccxt(exchange_config) + self._api = self._init_ccxt(exchange_config) logger.info('Using Exchange "%s"', self.get_name()) @@ -99,10 +96,10 @@ class Exchange(object): return api def get_name(self) -> str: - return self._API.name + return self._api.name def get_id(self) -> str: - return self._API.id + return self._api.id def validate_pairs(self, pairs: List[str]) -> None: """ @@ -113,7 +110,7 @@ class Exchange(object): """ try: - markets = self._API.load_markets() + markets = self._api.load_markets() except ccxt.BaseError as e: logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return @@ -136,7 +133,7 @@ class Exchange(object): :param endpoint: Name of endpoint (e.g. 'fetchOHLCV', 'fetchTickers') :return: bool """ - return endpoint in self._API.has and self._API.has[endpoint] + return endpoint in self._api.has and self._api.has[endpoint] def buy(self, pair: str, rate: float, amount: float) -> Dict: if self._CONF['dry_run']: @@ -155,7 +152,7 @@ class Exchange(object): return {'id': order_id} try: - return self._API.create_limit_buy_order(pair, amount, rate) + return self._api.create_limit_buy_order(pair, amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit buy order on market {pair}.' @@ -188,7 +185,7 @@ class Exchange(object): return {'id': order_id} try: - return self._API.create_limit_sell_order(pair, amount, rate) + return self._api.create_limit_sell_order(pair, amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit sell order on market {pair}.' @@ -224,7 +221,7 @@ class Exchange(object): return {} try: - balances = self._API.fetch_balance() + balances = self._api.fetch_balance() # Remove additional info from ccxt results balances.pop("info", None) balances.pop("free", None) @@ -241,10 +238,10 @@ class Exchange(object): @retrier def get_tickers(self) -> Dict: try: - return self._API.fetch_tickers() + return self._api.fetch_tickers() except ccxt.NotSupported as e: raise OperationalException( - f'Exchange {self._API.name} does not support fetching tickers in batch.' + f'Exchange {self._api.name} does not support fetching tickers in batch.' f'Message: {e}') except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( @@ -256,7 +253,7 @@ class Exchange(object): def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: if refresh or pair not in self._CACHED_TICKER.keys(): try: - data = self._API.fetch_ticker(pair) + data = self._api.fetch_ticker(pair) try: self._CACHED_TICKER[pair] = { 'bid': float(data['bid']), @@ -290,7 +287,7 @@ class Exchange(object): data: List[Dict[Any, Any]] = [] while not since_ms or since_ms < till_time_ms: - data_part = self._API.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + data_part = self._api.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) # Because some exchange sort Tickers ASC and other DESC. # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) @@ -311,7 +308,7 @@ class Exchange(object): return data except ccxt.NotSupported as e: raise OperationalException( - f'Exchange {self._API.name} does not support fetching historical candlestick data.' + f'Exchange {self._api.name} does not support fetching historical candlestick data.' f'Message: {e}') except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( @@ -325,7 +322,7 @@ class Exchange(object): return try: - return self._API.cancel_order(order_id, pair) + return self._api.cancel_order(order_id, pair) except ccxt.InvalidOrder as e: raise DependencyException( f'Could not cancel order. Message: {e}') @@ -344,7 +341,7 @@ class Exchange(object): }) return order try: - return self._API.fetch_order(order_id, pair) + return self._api.fetch_order(order_id, pair) except ccxt.InvalidOrder as e: raise DependencyException( f'Could not get order. Message: {e}') @@ -361,7 +358,7 @@ class Exchange(object): if not self.exchange_has('fetchMyTrades'): return [] try: - my_trades = self._API.fetch_my_trades(pair, since.timestamp()) + my_trades = self._api.fetch_my_trades(pair, since.timestamp()) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] return matched_trades @@ -374,10 +371,10 @@ class Exchange(object): def get_pair_detail_url(self, pair: str) -> str: try: - url_base = self._API.urls.get('www') + url_base = self._api.urls.get('www') base, quote = pair.split('/') - return url_base + _EXCHANGE_URLS[self._API.id].format(base=base, quote=quote) + return url_base + _EXCHANGE_URLS[self._api.id].format(base=base, quote=quote) except KeyError: logger.warning('Could not get exchange url for %s', self.get_name()) return "" @@ -385,7 +382,7 @@ class Exchange(object): @retrier def get_markets(self) -> List[dict]: try: - return self._API.fetch_markets() + return self._api.fetch_markets() except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not load markets due to {e.__class__.__name__}. Message: {e}') @@ -397,10 +394,10 @@ class Exchange(object): price=1, taker_or_maker='maker') -> float: try: # validate that markets are loaded before trying to get fee - if self._API.markets is None or len(self._API.markets) == 0: - self._API.load_markets() + if self._api.markets is None or len(self._api.markets) == 0: + self._api.load_markets() - return self._API.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, price=price, takerOrMaker=taker_or_maker)['rate'] except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( @@ -413,6 +410,6 @@ class Exchange(object): get buyable amount rounding, .. """ # validate that markets are loaded before trying to get fee - if not self._API.markets: - self._API.load_markets() - return self._API.amount_to_lots(pair, amount) + if not self._api.markets: + self._api.load_markets() + return self._api.amount_to_lots(pair, amount) From ef5313449908bcdd2f44d33eee49763b93b228ea Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 22:09:46 +0200 Subject: [PATCH 104/239] lowercase variables --- freqtrade/exchange/__init__.py | 36 +++++++++++------------ freqtrade/tests/exchange/test_exchange.py | 8 ++--- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 9f07d66d2..971a09bd8 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -44,11 +44,11 @@ class Exchange(object): # Current selected exchange _api: ccxt.Exchange = None - _CONF: Dict = {} - _CACHED_TICKER: Dict[str, Any] = {} + _conf: Dict = {} + _cached_ticker: Dict[str, Any] = {} # Holds all open sell orders for dry_run - _DRY_RUN_OPEN_ORDERS: Dict[str, Any] = {} + _dry_run_open_orders: Dict[str, Any] = {} def __init__(self, config: dict) -> None: """ @@ -57,7 +57,7 @@ class Exchange(object): exchange and pairs are valid. :return: None """ - self._CONF.update(config) + self._conf.update(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') @@ -115,7 +115,7 @@ class Exchange(object): logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return - stake_cur = self._CONF['stake_currency'] + stake_cur = self._conf['stake_currency'] for pair in pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format @@ -136,9 +136,9 @@ class Exchange(object): return endpoint in self._api.has and self._api.has[endpoint] def buy(self, pair: str, rate: float, amount: float) -> Dict: - if self._CONF['dry_run']: + if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' - self._DRY_RUN_OPEN_ORDERS[order_id] = { + self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, @@ -170,9 +170,9 @@ class Exchange(object): raise OperationalException(e) def sell(self, pair: str, rate: float, amount: float) -> Dict: - if self._CONF['dry_run']: + if self._conf['dry_run']: order_id = f'dry_run_sell_{randint(0, 10**6)}' - self._DRY_RUN_OPEN_ORDERS[order_id] = { + self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, @@ -204,7 +204,7 @@ class Exchange(object): @retrier def get_balance(self, currency: str) -> float: - if self._CONF['dry_run']: + if self._conf['dry_run']: return 999.9 # ccxt exception is already handled by get_balances @@ -217,7 +217,7 @@ class Exchange(object): @retrier def get_balances(self) -> dict: - if self._CONF['dry_run']: + if self._conf['dry_run']: return {} try: @@ -251,11 +251,11 @@ class Exchange(object): @retrier def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: - if refresh or pair not in self._CACHED_TICKER.keys(): + if refresh or pair not in self._cached_ticker.keys(): try: data = self._api.fetch_ticker(pair) try: - self._CACHED_TICKER[pair] = { + self._cached_ticker[pair] = { 'bid': float(data['bid']), 'ask': float(data['ask']), } @@ -269,7 +269,7 @@ class Exchange(object): raise OperationalException(e) else: logger.info("returning cached ticker-data for %s", pair) - return self._CACHED_TICKER[pair] + return self._cached_ticker[pair] @retrier def get_ticker_history(self, pair: str, tick_interval: str, @@ -318,7 +318,7 @@ class Exchange(object): @retrier def cancel_order(self, order_id: str, pair: str) -> None: - if self._CONF['dry_run']: + if self._conf['dry_run']: return try: @@ -334,8 +334,8 @@ class Exchange(object): @retrier def get_order(self, order_id: str, pair: str) -> Dict: - if self._CONF['dry_run']: - order = self._DRY_RUN_OPEN_ORDERS[order_id] + if self._conf['dry_run']: + order = self._dry_run_open_orders[order_id] order.update({ 'id': order_id }) @@ -353,7 +353,7 @@ class Exchange(object): @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: - if self._CONF['dry_run']: + if self._conf['dry_run']: return [] if not self.exchange_has('fetchMyTrades'): return [] diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 44bf539d0..43cd0076c 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -336,9 +336,9 @@ def test_get_ticker(default_conf, mocker): assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 - assert 'ETH/BTC' in exchange._CACHED_TICKER - assert exchange._CACHED_TICKER['ETH/BTC']['bid'] == 0.5 - assert exchange._CACHED_TICKER['ETH/BTC']['ask'] == 1 + assert 'ETH/BTC' in exchange._cached_ticker + assert exchange._cached_ticker['ETH/BTC']['bid'] == 0.5 + assert exchange._cached_ticker['ETH/BTC']['ask'] == 1 # Test caching api_mock.fetch_ticker = MagicMock() @@ -541,7 +541,7 @@ def test_get_order(default_conf, mocker): order = MagicMock() order.myid = 123 exchange = get_patched_exchange(mocker, default_conf) - exchange._DRY_RUN_OPEN_ORDERS['X'] = order + exchange._dry_run_open_orders['X'] = order print(exchange.get_order('X', 'TKN/BTC')) assert exchange.get_order('X', 'TKN/BTC').myid == 123 From 896afe711836bfb0285add761d16f20e5aadc511 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 22:20:50 +0200 Subject: [PATCH 105/239] convert get_name and get_id to properties --- freqtrade/exchange/__init__.py | 14 +++++++++----- freqtrade/freqtradebot.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 10 +++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 971a09bd8..481469701 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -65,7 +65,7 @@ class Exchange(object): exchange_config = config['exchange'] self._api = self._init_ccxt(exchange_config) - logger.info('Using Exchange "%s"', self.get_name()) + logger.info('Using Exchange "%s"', self.name) # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) @@ -95,10 +95,14 @@ class Exchange(object): return api - def get_name(self) -> str: + @property + def name(self) -> str: + """exchange Name (from ccxt)""" return self._api.name - def get_id(self) -> str: + @property + def id(self) -> str: + """exchange ccxt id""" return self._api.id def validate_pairs(self, pairs: List[str]) -> None: @@ -124,7 +128,7 @@ class Exchange(object): f'Pair {pair} not compatible with stake_currency: {stake_cur}') if pair not in markets: raise OperationalException( - f'Pair {pair} is not available at {self.get_name()}') + f'Pair {pair} is not available at {self.name}') def exchange_has(self, endpoint: str) -> bool: """ @@ -376,7 +380,7 @@ class Exchange(object): return url_base + _EXCHANGE_URLS[self._api.id].format(base=base, quote=quote) except KeyError: - logger.warning('Could not get exchange url for %s', self.get_name()) + logger.warning('Could not get exchange url for %s', self.name) return "" @retrier diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index dd35700a4..02375ca80 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -254,7 +254,7 @@ class FreqtradeBot(object): interval = self.analyze.get_ticker_interval() stake_currency = self.config['stake_currency'] fiat_currency = self.config['fiat_display_currency'] - exc_name = self.exchange.get_name() + exc_name = self.exchange.name logger.info( 'Checking buy signals to create a new trade with stake_amount: %f ...', @@ -314,7 +314,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ open_rate=buy_limit, open_rate_requested=buy_limit, open_date=datetime.utcnow(), - exchange=self.exchange.get_id(), + exchange=self.exchange.id, open_order_id=order_id ) Trade.session.add(trade) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 43cd0076c..3c68e8493 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -66,7 +66,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker): def test_validate_pairs_exception(default_conf, mocker, caplog): caplog.set_level(logging.INFO) api_mock = MagicMock() - mocker.patch('freqtrade.exchange.Exchange.get_name', MagicMock(return_value='Binance')) + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) @@ -570,21 +570,21 @@ def test_get_order(default_conf, mocker): assert api_mock.fetch_order.call_count == 1 -def test_get_name(default_conf, mocker): +def test_name(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_pairs', side_effect=lambda s: True) default_conf['exchange']['name'] = 'binance' exchange = Exchange(default_conf) - assert exchange.get_name() == 'Binance' + assert exchange.name == 'Binance' -def test_get_id(default_conf, mocker): +def test_id(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_pairs', side_effect=lambda s: True) default_conf['exchange']['name'] = 'binance' exchange = Exchange(default_conf) - assert exchange.get_id() == 'binance' + assert exchange.id == 'binance' def test_get_pair_detail_url(default_conf, mocker, caplog): From 2b0ef5459546a99a7e753cf55427490493fc8c99 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 22:28:51 +0200 Subject: [PATCH 106/239] update download_script for exchange objectify --- scripts/download_backtest_data.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 9aedbecb9..2f76c1232 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -6,7 +6,8 @@ import sys import os import arrow -from freqtrade import (exchange, arguments, misc) +from freqtrade import (arguments, misc) +from freqtrade.exchange import Exchange DEFAULT_DL_PATH = 'user_data/data' @@ -39,16 +40,21 @@ if args.days: print(f'About to download pairs: {PAIRS} to {dl_path}') + # Init exchange -exchange._API = exchange.init_ccxt({'key': '', - 'secret': '', - 'name': args.exchange}) +exchange = Exchange({'key': '', + 'secret': '', + 'stake_currency': '', + 'dry_run': True, + 'exchange': { + 'name': args.exchange, + 'pair_whitelist': [] + } + }) pairs_not_available = [] -# Make sure API markets is initialized -exchange._API.load_markets() for pair in PAIRS: - if pair not in exchange._API.markets: + if pair not in exchange._api.markets: pairs_not_available.append(pair) print(f"skipping pair {pair}") continue From 488f1717a19649e06c298d5dae5ca8d16ca740c0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 22:32:29 +0200 Subject: [PATCH 107/239] update plot_dataframe script to objectify exchange --- scripts/plot_dataframe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ce1a4b819..7f9641222 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -35,10 +35,10 @@ from plotly import tools from plotly.offline import plot import freqtrade.optimize as optimize -from freqtrade import exchange from freqtrade import persistence from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments +from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade @@ -73,7 +73,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Load the strategy try: analyze = Analyze(_CONF) - exchange.init(_CONF) + exchange = Exchange(_CONF) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', From f7b46d5404ece4e430377a005cf71930b3d47848 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 22:34:28 +0200 Subject: [PATCH 108/239] update docstring --- freqtrade/exchange/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 481469701..acfefdad4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -74,8 +74,6 @@ class Exchange(object): """ Initialize ccxt with given config and return valid ccxt instance. - :param config: config to use - :return: ccxt """ # Find matching class for the given exchange name name = exchange_config['name'] From a7be15d72f9a3d534042040fdf19e1a3f4974c57 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 18 Jun 2018 22:42:14 +0200 Subject: [PATCH 109/239] Update Documentation to include backtesting with docker --- docs/installation.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index b6688885b..87d3dc5cf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -184,6 +184,26 @@ docker start freqtrade You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. +### 7. Backtest with docker + +The following assumes that the above steps (1-4) have been completed successfully. +Also, backtest-data should be available at `~/.freqtrade/user_data/`. + + +``` bash +docker run -d \ + --name freqtrade \ + -v /etc/localtime:/etc/localtime:ro \ + -v ~/.freqtrade/config.json:/freqtrade/config.json \ + -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ + -v ~/.freqtrade/user_data/:/freqtrade/user_data/ \ + freqtrade --strategy AwsomelyProfitableStrategy backtesting +``` + +Head over to the [Backtesting Documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) for more details. + +*Note*: Additional parameters can be appended after the image name (`freqtrade` in the above example). + ------ ## Custom Installation From e66b861c9e93cec7079dbca6ab09ee7d1febeedb Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 19 Jun 2018 14:23:05 +0200 Subject: [PATCH 110/239] Update ccxt from 1.14.211 to 1.14.224 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 96f3169c8..104634890 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.211 +ccxt==1.14.224 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From a493a2ceef49a5420e6c883a50f6326f45f2c70d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 20 Jun 2018 14:23:06 +0200 Subject: [PATCH 111/239] Update ccxt from 1.14.224 to 1.14.230 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 104634890..ed01a35d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.224 +ccxt==1.14.230 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 36cfea3d0f413ab51d9042bc461c44617fdc93d2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 20 Jun 2018 14:23:08 +0200 Subject: [PATCH 112/239] Update pytest from 3.6.1 to 3.6.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed01a35d4..a86cdc020 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.14.5 TA-Lib==0.4.17 -pytest==3.6.1 +pytest==3.6.2 pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 From c7976f51e20ff2bf2e24cb26e455d843eaa44c16 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 21 Jun 2018 14:24:06 +0200 Subject: [PATCH 113/239] Update ccxt from 1.14.230 to 1.14.242 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a86cdc020..41e246d50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.230 +ccxt==1.14.242 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 7f927b4d7a636feb36a7e28a348b36a5d1ee6e32 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 21 Jun 2018 20:47:53 +0200 Subject: [PATCH 114/239] Squashed commit of the following: commit 435f299bcf52c7fe0be1b44c5ea7a9983f31b811 Author: Gert Wohlgemuth Date: Wed Jun 20 01:57:28 2018 -0700 improve readability of outdated history code --- freqtrade/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index f18ae291c..17a8f27b9 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -148,7 +148,7 @@ class Analyze(object): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < arrow.utcnow() - timedelta(minutes=(interval_minutes + 5)): + if signal_date < (arrow.utcnow() - timedelta(minutes=(interval_minutes + 5))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', pair, From 98cd8970f996ca64f96ca32193fe746ec1bddd40 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 22 Jun 2018 14:24:06 +0200 Subject: [PATCH 115/239] Update ccxt from 1.14.242 to 1.14.253 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 41e246d50..a31976c8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.242 +ccxt==1.14.253 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From c73b9f5c77ab9f55b9d51326f9c372dc02af02b1 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:04:07 +0300 Subject: [PATCH 116/239] avoid calling exchange.get_fee inside loop --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9a19d1412..ffb808a24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -62,6 +62,7 @@ class Backtesting(object): self.config['exchange']['uid'] = '' self.config['dry_run'] = True self.exchange = Exchange(self.config) + self.fee = self.exchange.get_fee() @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -130,14 +131,13 @@ class Backtesting(object): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) - fee = self.exchange.get_fee() trade = Trade( open_rate=buy_row.close, open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, - fee_open=fee, - fee_close=fee + fee_open=self.fee, + fee_close=self.fee ) # calculate win/lose forwards from buy point From 8a44dff595da271b35e8fbc4696f15bd8271b80e Mon Sep 17 00:00:00 2001 From: xmatthias Date: Fri, 22 Jun 2018 20:10:05 +0200 Subject: [PATCH 117/239] don't sell if buy is still active --- config.json.example | 3 +- config_full.json.example | 3 +- docs/configuration.md | 1 + freqtrade/analyze.py | 4 ++ freqtrade/constants.py | 3 +- freqtrade/freqtradebot.py | 3 +- freqtrade/tests/test_freqtradebot.py | 77 ++++++++++++++++++++++++++++ 7 files changed, 90 insertions(+), 4 deletions(-) diff --git a/config.json.example b/config.json.example index d3dbeb52e..e5bdc95b1 100644 --- a/config.json.example +++ b/config.json.example @@ -31,7 +31,8 @@ }, "experimental": { "use_sell_signal": false, - "sell_profit_only": false + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index c17d22a15..231384acc 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -38,7 +38,8 @@ }, "experimental": { "use_sell_signal": false, - "sell_profit_only": false + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false }, "telegram": { "enabled": true, diff --git a/docs/configuration.md b/docs/configuration.md index d5d53860b..c7ba9febe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,6 +31,7 @@ The table below will list all configuration parameters. | `exchange.pair_blacklist` | [] | No | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. +| `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index cf2d5519b..b986b9611 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -172,6 +172,10 @@ class Analyze(object): if the threshold is reached and updates the trade record. :return: True if trade should be sold, False otherwise """ + if buy and self.config.get('experimental', {}).get('ignore_roi_if_buy_signal', False): + logger.debug('Buy signal still active - not selling.') + return False + # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) if self.min_roi_reached(trade=trade, current_rate=rate, current_time=date): logger.debug('Required profit reached. Selling..') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5be01f977..bf661aecc 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -73,7 +73,8 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'} + 'sell_profit_only': {'type': 'boolean'}, + "ignore_roi_if_buy_signal_true": {'type': 'boolean'} } }, 'telegram': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 02375ca80..f06460706 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -424,7 +424,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ (buy, sell) = (False, False) - if self.config.get('experimental', {}).get('use_sell_signal'): + if (self.config.get('experimental', {}).get('use_sell_signal') + or self.config.get('experimental', {}).get('ignore_roi_if_buy_signal')): (buy, sell) = self.analyze.get_signal(self.exchange, trade.pair, self.analyze.get_ticker_interval()) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9eae2fdf5..c18950177 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1335,6 +1335,83 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke assert freqtrade.handle_trade(trade) is True +def ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['experimental'] = { + 'ignore_roi_if_buy_signal': True + } + + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(True, True)) + assert freqtrade.handle_trade(trade) is False + + # Test if buy-signal is absent (should sell due to roi = true) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) is True + + +def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['experimental'] = { + 'ignore_roi_if_buy_signal': False + } + + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + # Sell due to min_roi_reached + patch_get_signal(mocker, value=(True, True)) + assert freqtrade.handle_trade(trade) is True + + # Test if buy-signal is absent + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) is True + + def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker): """ Test get_real_amount - fee in quote currency From cbfee51f321992ba17501605514e87cf8a6361f3 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Fri, 22 Jun 2018 20:51:21 +0200 Subject: [PATCH 118/239] introduce experimental variable and fix test naming --- freqtrade/analyze.py | 9 ++++----- freqtrade/freqtradebot.py | 5 ++--- freqtrade/tests/test_freqtradebot.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b986b9611..fc81f3fb9 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -172,7 +172,8 @@ class Analyze(object): if the threshold is reached and updates the trade record. :return: True if trade should be sold, False otherwise """ - if buy and self.config.get('experimental', {}).get('ignore_roi_if_buy_signal', False): + experimental = self.config.get('experimental', {}) + if buy and experimental.get('ignore_roi_if_buy_signal', False): logger.debug('Buy signal still active - not selling.') return False @@ -181,13 +182,11 @@ class Analyze(object): logger.debug('Required profit reached. Selling..') return True - # Experimental: Check if the trade is profitable before selling it (avoid selling at loss) - if self.config.get('experimental', {}).get('sell_profit_only', False): + if experimental.get('sell_profit_only', False): logger.debug('Checking if trade is profitable..') if trade.calc_profit(rate=rate) <= 0: return False - - if sell and not buy and self.config.get('experimental', {}).get('use_sell_signal', False): + if sell and not buy and experimental.get('use_sell_signal', False): logger.debug('Sell signal received. Selling..') return True diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f06460706..221d32e9e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -423,9 +423,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ current_rate = self.exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) - - if (self.config.get('experimental', {}).get('use_sell_signal') - or self.config.get('experimental', {}).get('ignore_roi_if_buy_signal')): + experimental = self.config.get('experimental', {}) + if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): (buy, sell) = self.analyze.get_signal(self.exchange, trade.pair, self.analyze.get_ticker_interval()) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c18950177..bad6d9502 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1335,7 +1335,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke assert freqtrade.handle_trade(trade) is True -def ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ From f7e5d2c3a572d7f1af3f28c164341ba942d480d9 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:55:09 +0300 Subject: [PATCH 119/239] check that we set fee on backtesting init --- freqtrade/tests/optimize/test_backtesting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 1702818b1..22536e42f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -298,6 +298,7 @@ def test_backtesting_init(mocker, default_conf) -> None: Test Backtesting._init() method """ patch_exchange(mocker) + get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert isinstance(backtesting.analyze, Analyze) @@ -305,6 +306,8 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.tickerdata_to_dataframe) assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) + get_fee.assert_called() + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From e2a2a0be9b7bed68504d7adac5048c4c41d5980a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Fri, 22 Jun 2018 21:21:34 +0200 Subject: [PATCH 120/239] extract stop_loss_reached to allow check before ignore_roi_if_buy_signal --- freqtrade/analyze.py | 20 ++++++++++++++------ freqtrade/tests/test_freqtradebot.py | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index fc81f3fb9..4f4b4f391 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -172,13 +172,17 @@ class Analyze(object): if the threshold is reached and updates the trade record. :return: True if trade should be sold, False otherwise """ + current_profit = trade.calc_profit_percent(rate) experimental = self.config.get('experimental', {}) + if self.stop_loss_reached(current_profit=current_profit): + return True + if buy and experimental.get('ignore_roi_if_buy_signal', False): logger.debug('Buy signal still active - not selling.') return False # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) - if self.min_roi_reached(trade=trade, current_rate=rate, current_time=date): + if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): logger.debug('Required profit reached. Selling..') return True @@ -192,16 +196,20 @@ class Analyze(object): return False - def min_roi_reached(self, trade: Trade, current_rate: float, current_time: datetime) -> bool: + def stop_loss_reached(self, current_profit: float) -> bool: + """Based on current profit of the trade and configured stoploss, decides to sell or not""" + + if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: + logger.debug('Stop loss hit.') + return True + return False + + def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: """ Based an earlier trade and current price and ROI configuration, decides whether bot should sell :return True if bot should sell at current rate """ - current_profit = trade.calc_profit_percent(current_rate) - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: - logger.debug('Stop loss hit.') - return True # Check if time matches and current rate is above threshold time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index bad6d9502..33d9ae2d9 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1274,7 +1274,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch('freqtrade.freqtradebot.Analyze.stop_loss_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), From 2be7b3d9eb71b08543b5757947ded059e9eb4e4c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Fri, 22 Jun 2018 21:24:21 +0200 Subject: [PATCH 121/239] fix mocked bid-value to match limt_buy_order config --- freqtrade/tests/test_freqtradebot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 33d9ae2d9..0d4256c42 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1312,9 +1312,9 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1347,9 +1347,9 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, From 9a07d57ed7b8f45352cf78ae7f6d45cb01acfa54 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 07:58:25 +0300 Subject: [PATCH 122/239] fix flake8 --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 22536e42f..65aa00a70 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -307,7 +307,7 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) get_fee.assert_called() - assert backtesting.fee == 0.5 + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From 3360bf400125cf4d058534e9edfacd8ac3e54c1e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 10:25:03 +0200 Subject: [PATCH 123/239] wrap strategies with HyperoptStrategy for module lookups with pickle --- freqtrade/optimize/hyperopt.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 19e732d7b..e52581491 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,6 +11,7 @@ import pickle import signal import sys from argparse import Namespace +from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -26,10 +27,25 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) +HyperoptStrategy = None + + +def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: + """Wraps a given Strategy instance to HyperoptStrategy""" + global HyperoptStrategy + + attr = deepcopy(dict(strategy.__class__.__dict__)) + # Patch module name to make it compatible with pickle + attr['__module__'] = 'freqtrade.optimize.hyperopt' + HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) + return HyperoptStrategy() + + class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -39,7 +55,6 @@ class Hyperopt(Backtesting): hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days @@ -57,6 +72,9 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 + # Wrap strategy to make it compatible with pickle + self.analyze.strategy = wrap_strategy(self.analyze.strategy) + # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None From c40e6a12d1ca4f302f876c10a3a107dcb5aae1c9 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:13:49 +0200 Subject: [PATCH 124/239] move logic from hyperopt to freqtrade.strategy --- freqtrade/optimize/hyperopt.py | 19 ------------------- freqtrade/strategy/__init__.py | 32 ++++++++++++++++++++++++++++++++ freqtrade/strategy/resolver.py | 5 +++-- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e52581491..7a313a3ac 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,7 +11,6 @@ import pickle import signal import sys from argparse import Namespace -from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -27,25 +26,10 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) -HyperoptStrategy = None - - -def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: - """Wraps a given Strategy instance to HyperoptStrategy""" - global HyperoptStrategy - - attr = deepcopy(dict(strategy.__class__.__dict__)) - # Patch module name to make it compatible with pickle - attr['__module__'] = 'freqtrade.optimize.hyperopt' - HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) - return HyperoptStrategy() - - class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -72,9 +56,6 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 - # Wrap strategy to make it compatible with pickle - self.analyze.strategy = wrap_strategy(self.analyze.strategy) - # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index e69de29bb..e1dc7bb3f 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -0,0 +1,32 @@ +import logging +from copy import deepcopy + +from freqtrade.strategy.interface import IStrategy + + +logger = logging.getLogger(__name__) + + +def import_strategy(strategy: IStrategy) -> IStrategy: + """ + Imports given Strategy instance to global scope + of freqtrade.strategy and returns an instance of it + """ + # Copy all attributes from base class and class + attr = deepcopy({**strategy.__class__.__dict__, **strategy.__dict__}) + # Adjust module name + attr['__module__'] = 'freqtrade.strategy' + + name = strategy.__class__.__name__ + clazz = type(name, (IStrategy,), attr) + + logger.debug( + 'Imported strategy %s.%s as %s.%s', + strategy.__module__, strategy.__class__.__name__, + clazz.__module__, strategy.__class__.__name__, + ) + + # Modify global scope to declare class + globals()[name] = clazz + + return clazz() diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3fd39bca3..3c7836291 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -11,6 +11,7 @@ from collections import OrderedDict from typing import Optional, Dict, Type from freqtrade import constants +from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy @@ -83,7 +84,7 @@ class StrategyResolver(object): strategy = self._search_strategy(path, strategy_name) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return strategy + return import_strategy(strategy) raise ImportError( "Impossible to load Strategy '{}'. This class does not exist" @@ -100,7 +101,7 @@ class StrategyResolver(object): """ # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) + spec = importlib.util.spec_from_file_location('unknown', module_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore # importlib does not use typehints From 4bd61df3a7e7405c821e38f5bce4f06ae2fe4917 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:14:31 +0200 Subject: [PATCH 125/239] implement test for import_strategy --- freqtrade/tests/strategy/test_strategy.py | 27 +++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 244910790..26cd798f4 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -5,10 +5,34 @@ import os import pytest +from freqtrade.strategy import import_strategy +from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver +def test_import_strategy(caplog): + caplog.set_level(logging.DEBUG) + + strategy = DefaultStrategy() + strategy.some_method = lambda *args, **kwargs: 42 + + assert strategy.__module__ == 'freqtrade.strategy.default_strategy' + assert strategy.some_method() == 42 + + imported_strategy = import_strategy(strategy) + + assert imported_strategy.__module__ == 'freqtrade.strategy' + assert imported_strategy.some_method() == 42 + + assert ( + 'freqtrade.strategy', + logging.DEBUG, + 'Imported strategy freqtrade.strategy.default_strategy.DefaultStrategy ' + 'as freqtrade.strategy.DefaultStrategy', + ) in caplog.record_tuples + + def test_search_strategy(): default_location = os.path.join(os.path.dirname( os.path.realpath(__file__)), '..', '..', 'strategy' @@ -20,8 +44,7 @@ def test_search_strategy(): def test_load_strategy(result): - resolver = StrategyResolver() - resolver._load_strategy('TestStrategy') + resolver = StrategyResolver({'strategy': 'TestStrategy'}) assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) From 818a6b12ed6cad47a4b7e758534b7c5721365e06 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:57:26 +0200 Subject: [PATCH 126/239] tests: add dir() assertion --- freqtrade/tests/strategy/test_strategy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 26cd798f4..13eac3261 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 - import logging import os @@ -22,6 +21,8 @@ def test_import_strategy(caplog): imported_strategy = import_strategy(strategy) + assert dir(strategy) == dir(imported_strategy) + assert imported_strategy.__module__ == 'freqtrade.strategy' assert imported_strategy.some_method() == 42 From fc219b4e940d84e1d0e45efb5b48bc1a2631858f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 13:10:08 +0200 Subject: [PATCH 127/239] move experimental eval below stop_loss_reached to improve performance --- freqtrade/analyze.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 4f4b4f391..a0f133b22 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -173,10 +173,11 @@ class Analyze(object): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - experimental = self.config.get('experimental', {}) if self.stop_loss_reached(current_profit=current_profit): return True + experimental = self.config.get('experimental', {}) + if buy and experimental.get('ignore_roi_if_buy_signal', False): logger.debug('Buy signal still active - not selling.') return False From d23cd73ba82abb0d670ccdbccfb4dcd821a557c9 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 13:12:36 +0200 Subject: [PATCH 128/239] update plotly --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a31976c8c..e26c9c8f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Required for plotting data -#plotly==2.3.0 +#plotly==2.7.0 From a68c90c51229d9586cd5ff38cbf565006cc01984 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:04:07 +0300 Subject: [PATCH 129/239] avoid calling exchange.get_fee inside loop --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9a19d1412..ffb808a24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -62,6 +62,7 @@ class Backtesting(object): self.config['exchange']['uid'] = '' self.config['dry_run'] = True self.exchange = Exchange(self.config) + self.fee = self.exchange.get_fee() @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -130,14 +131,13 @@ class Backtesting(object): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) - fee = self.exchange.get_fee() trade = Trade( open_rate=buy_row.close, open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, - fee_open=fee, - fee_close=fee + fee_open=self.fee, + fee_close=self.fee ) # calculate win/lose forwards from buy point From c1691f21f3e3887d5842b091656084cc07de9088 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:55:09 +0300 Subject: [PATCH 130/239] check that we set fee on backtesting init --- freqtrade/tests/optimize/test_backtesting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 1702818b1..22536e42f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -298,6 +298,7 @@ def test_backtesting_init(mocker, default_conf) -> None: Test Backtesting._init() method """ patch_exchange(mocker) + get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert isinstance(backtesting.analyze, Analyze) @@ -305,6 +306,8 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.tickerdata_to_dataframe) assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) + get_fee.assert_called() + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From 01d45bee76addbec4b66c999ae12d17409d9016e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 07:58:25 +0300 Subject: [PATCH 131/239] fix flake8 --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 22536e42f..65aa00a70 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -307,7 +307,7 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) get_fee.assert_called() - assert backtesting.fee == 0.5 + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From 2738d3aed85ea5088af595751590a5000a190c68 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 13:12:36 +0200 Subject: [PATCH 132/239] update plotly --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a31976c8c..e26c9c8f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Required for plotting data -#plotly==2.3.0 +#plotly==2.7.0 From 5aae215c94cbe37b3981e51a6b460e36a7fbfa8e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 10:25:03 +0200 Subject: [PATCH 133/239] wrap strategies with HyperoptStrategy for module lookups with pickle --- freqtrade/optimize/hyperopt.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 19e732d7b..e52581491 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,6 +11,7 @@ import pickle import signal import sys from argparse import Namespace +from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -26,10 +27,25 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) +HyperoptStrategy = None + + +def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: + """Wraps a given Strategy instance to HyperoptStrategy""" + global HyperoptStrategy + + attr = deepcopy(dict(strategy.__class__.__dict__)) + # Patch module name to make it compatible with pickle + attr['__module__'] = 'freqtrade.optimize.hyperopt' + HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) + return HyperoptStrategy() + + class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -39,7 +55,6 @@ class Hyperopt(Backtesting): hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days @@ -57,6 +72,9 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 + # Wrap strategy to make it compatible with pickle + self.analyze.strategy = wrap_strategy(self.analyze.strategy) + # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None From 78f50a14713c70c1b444e9c2ec0ec7569230f25e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:13:49 +0200 Subject: [PATCH 134/239] move logic from hyperopt to freqtrade.strategy --- freqtrade/optimize/hyperopt.py | 19 ------------------- freqtrade/strategy/__init__.py | 32 ++++++++++++++++++++++++++++++++ freqtrade/strategy/resolver.py | 5 +++-- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e52581491..7a313a3ac 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,7 +11,6 @@ import pickle import signal import sys from argparse import Namespace -from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -27,25 +26,10 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) -HyperoptStrategy = None - - -def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: - """Wraps a given Strategy instance to HyperoptStrategy""" - global HyperoptStrategy - - attr = deepcopy(dict(strategy.__class__.__dict__)) - # Patch module name to make it compatible with pickle - attr['__module__'] = 'freqtrade.optimize.hyperopt' - HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) - return HyperoptStrategy() - - class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -72,9 +56,6 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 - # Wrap strategy to make it compatible with pickle - self.analyze.strategy = wrap_strategy(self.analyze.strategy) - # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index e69de29bb..e1dc7bb3f 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -0,0 +1,32 @@ +import logging +from copy import deepcopy + +from freqtrade.strategy.interface import IStrategy + + +logger = logging.getLogger(__name__) + + +def import_strategy(strategy: IStrategy) -> IStrategy: + """ + Imports given Strategy instance to global scope + of freqtrade.strategy and returns an instance of it + """ + # Copy all attributes from base class and class + attr = deepcopy({**strategy.__class__.__dict__, **strategy.__dict__}) + # Adjust module name + attr['__module__'] = 'freqtrade.strategy' + + name = strategy.__class__.__name__ + clazz = type(name, (IStrategy,), attr) + + logger.debug( + 'Imported strategy %s.%s as %s.%s', + strategy.__module__, strategy.__class__.__name__, + clazz.__module__, strategy.__class__.__name__, + ) + + # Modify global scope to declare class + globals()[name] = clazz + + return clazz() diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3fd39bca3..3c7836291 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -11,6 +11,7 @@ from collections import OrderedDict from typing import Optional, Dict, Type from freqtrade import constants +from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy @@ -83,7 +84,7 @@ class StrategyResolver(object): strategy = self._search_strategy(path, strategy_name) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return strategy + return import_strategy(strategy) raise ImportError( "Impossible to load Strategy '{}'. This class does not exist" @@ -100,7 +101,7 @@ class StrategyResolver(object): """ # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) + spec = importlib.util.spec_from_file_location('unknown', module_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore # importlib does not use typehints From 398b21a11d175d02622bc3fb88ad38e52e04bbaf Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:14:31 +0200 Subject: [PATCH 135/239] implement test for import_strategy --- freqtrade/tests/strategy/test_strategy.py | 27 +++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 244910790..26cd798f4 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -5,10 +5,34 @@ import os import pytest +from freqtrade.strategy import import_strategy +from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver +def test_import_strategy(caplog): + caplog.set_level(logging.DEBUG) + + strategy = DefaultStrategy() + strategy.some_method = lambda *args, **kwargs: 42 + + assert strategy.__module__ == 'freqtrade.strategy.default_strategy' + assert strategy.some_method() == 42 + + imported_strategy = import_strategy(strategy) + + assert imported_strategy.__module__ == 'freqtrade.strategy' + assert imported_strategy.some_method() == 42 + + assert ( + 'freqtrade.strategy', + logging.DEBUG, + 'Imported strategy freqtrade.strategy.default_strategy.DefaultStrategy ' + 'as freqtrade.strategy.DefaultStrategy', + ) in caplog.record_tuples + + def test_search_strategy(): default_location = os.path.join(os.path.dirname( os.path.realpath(__file__)), '..', '..', 'strategy' @@ -20,8 +44,7 @@ def test_search_strategy(): def test_load_strategy(result): - resolver = StrategyResolver() - resolver._load_strategy('TestStrategy') + resolver = StrategyResolver({'strategy': 'TestStrategy'}) assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) From 810d7de8693ddbc4e0d3acf14c7684e9f51cd9dc Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:57:26 +0200 Subject: [PATCH 136/239] tests: add dir() assertion --- freqtrade/tests/strategy/test_strategy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 26cd798f4..13eac3261 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 - import logging import os @@ -22,6 +21,8 @@ def test_import_strategy(caplog): imported_strategy = import_strategy(strategy) + assert dir(strategy) == dir(imported_strategy) + assert imported_strategy.__module__ == 'freqtrade.strategy' assert imported_strategy.some_method() == 42 From b485e6e0ba151d23a41d5d0bcbf11be9e22ae827 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Jun 2018 22:40:36 +0300 Subject: [PATCH 137/239] start small --- freqtrade/optimize/hyperopt.py | 185 ++------------------------------- 1 file changed, 6 insertions(+), 179 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7a313a3ac..31764fbd7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -21,6 +21,8 @@ import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame +from skopt.space import Real, Integer, Categorical + import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -65,121 +67,18 @@ class Hyperopt(Backtesting): @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - """ dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] return dataframe @@ -295,38 +194,6 @@ class Hyperopt(Backtesting): {'enabled': False}, {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), } def has_space(self, space: str) -> bool: @@ -361,12 +228,8 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) if 'mfi' in params and params['mfi']['enabled']: conditions.append(dataframe['mfi'] < params['mfi']['value']) if 'fastd' in params and params['fastd']['enabled']: @@ -375,49 +238,13 @@ class Hyperopt(Backtesting): conditions.append(dataframe['adx'] > params['adx']['value']) if 'rsi' in params and params['rsi']['enabled']: conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) # TRIGGERS triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), } - conditions.append(triggers.get(params['trigger']['type'])) + #conditions.append(triggers.get(params['trigger']['type'])) + + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger dataframe.loc[ reduce(lambda x, y: x & y, conditions), From 0cb1aedf5bd7a6f746d68319d8f6bff327efbf2d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 19 Jun 2018 09:09:54 +0300 Subject: [PATCH 138/239] problem with pickling --- freqtrade/optimize/hyperopt.py | 157 +++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 66 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 31764fbd7..07e618b15 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,6 +10,8 @@ import os import pickle import signal import sys +import multiprocessing + from argparse import Namespace from functools import reduce from math import exp @@ -22,6 +24,8 @@ from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame from skopt.space import Real, Integer, Categorical +from skopt import Optimizer +from sklearn.externals.joblib import Parallel, delayed import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments @@ -65,6 +69,21 @@ class Hyperopt(Backtesting): self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') self.trials = Trials() + def get_args(self, params): + dimensions = self.hyperopt_space() + # Ensure the number of dimensions match + # the number of parameters in the list x. + if len(params) != len(dimensions): + msg = "Mismatch in number of search-space dimensions. " \ + "len(dimensions)=={} and len(x)=={}" + msg = msg.format(len(dimensions), len(params)) + raise ValueError(msg) + + # Create a dict where the keys are the names of the dimensions + # and the values are taken from the list of parameters x. + arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} + return arg_dict + @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) @@ -173,28 +192,17 @@ class Hyperopt(Backtesting): """ Define your Hyperopt space for searching strategy parameters """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - } + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + ] + def has_space(self, space: str) -> bool: """ @@ -208,14 +216,15 @@ class Hyperopt(Backtesting): """ Return the space to use during Hyperopt """ - spaces: Dict = {} - if self.has_space('buy'): - spaces = {**spaces, **Hyperopt.indicator_space()} - if self.has_space('roi'): - spaces = {**spaces, **Hyperopt.roi_space()} - if self.has_space('stoploss'): - spaces = {**spaces, **Hyperopt.stoploss_space()} - return spaces + return Hyperopt.indicator_space() + # spaces: Dict = {} + # if self.has_space('buy'): + # spaces = {**spaces, **Hyperopt.indicator_space()} + # if self.has_space('roi'): + # spaces = {**spaces, **Hyperopt.roi_space()} + # if self.has_space('stoploss'): + # spaces = {**spaces, **Hyperopt.stoploss_space()} + # return spaces @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: @@ -228,16 +237,16 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) +# if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: +# conditions.append(dataframe['macd'] < 0) + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS triggers = { @@ -254,7 +263,9 @@ class Hyperopt(Backtesting): return populate_buy_trend - def generate_optimizer(self, params: Dict) -> Dict: + def generate_optimizer(self, _params) -> Dict: + params = self.get_args(_params) + if self.has_space('roi'): self.analyze.strategy.minimal_roi = self.generate_roi_table(params) @@ -297,12 +308,13 @@ class Hyperopt(Backtesting): 'result': result_explanation, } ) + return loss - return { - 'loss': loss, - 'status': STATUS_OK, - 'result': result_explanation, - } +# return { +# 'loss': loss, +# 'status': STATUS_OK, +# 'result': result_explanation, +# } def format_results(self, results: DataFrame) -> str: """ @@ -347,16 +359,29 @@ class Hyperopt(Backtesting): ) try: - best_parameters = fmin( - fn=self.generate_optimizer, - space=self.hyperopt_space(), - algo=tpe.suggest, - max_evals=self.total_tries, - trials=self.trials - ) + # best_parameters = fmin( + # fn=self.generate_optimizer, + # space=self.hyperopt_space(), + # algo=tpe.suggest, + # max_evals=self.total_tries, + # trials=self.trials + # ) + + # results = sorted(self.trials.results, key=itemgetter('loss')) + # best_result = results[0]['result'] + cpus = multiprocessing.cpu_count() + print(f'Found {cpus}. Let\'s make them scream!') + + opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") + + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + #asked = opt.ask() + #f_val = self.generate_optimizer(asked) + f_val = Parallel(n_jobs=-1)(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, f_val) + print(f'got value {f_val}') - results = sorted(self.trials.results, key=itemgetter('loss')) - best_result = results[0]['result'] except ValueError: best_parameters = {} @@ -364,20 +389,20 @@ class Hyperopt(Backtesting): 'try with more epochs (param: -e).' # Improve best parameter logging display - if best_parameters: - best_parameters = space_eval( - self.hyperopt_space(), - best_parameters - ) + # if best_parameters: + # best_parameters = space_eval( + # self.hyperopt_space(), + # best_parameters + # ) - logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - if 'roi_t1' in best_parameters: - logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + # logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + # if 'roi_t1' in best_parameters: + # logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) - logger.info('Best Result:\n%s', best_result) + # logger.info('Best Result:\n%s', best_result) - # Store trials result to file to resume next time - self.save_trials() + # # Store trials result to file to resume next time + # self.save_trials() def signal_handler(self, sig, frame) -> None: """ From a46badd5c066f13bbbad1c38bf309c6346e965bf Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 19 Jun 2018 21:57:42 +0300 Subject: [PATCH 139/239] reuse pool workers --- freqtrade/optimize/hyperopt.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07e618b15..e49e66137 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -308,13 +308,12 @@ class Hyperopt(Backtesting): 'result': result_explanation, } ) - return loss -# return { -# 'loss': loss, -# 'status': STATUS_OK, -# 'result': result_explanation, -# } + return { + 'loss': loss, + 'status': STATUS_OK, + 'result': result_explanation, + } def format_results(self, results: DataFrame) -> str: """ @@ -374,13 +373,14 @@ class Hyperopt(Backtesting): opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - #asked = opt.ask() - #f_val = self.generate_optimizer(asked) - f_val = Parallel(n_jobs=-1)(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, f_val) - print(f'got value {f_val}') + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + #asked = opt.ask() + #f_val = self.generate_optimizer(asked) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) + print(f'got value {f_val}') except ValueError: From 964cbdc262115772b7fdb6f2322de0c93facfc0c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 20 Jun 2018 09:13:51 +0300 Subject: [PATCH 140/239] increase initial sampling points --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e49e66137..919526e4a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -371,7 +371,7 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") + opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30) with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From c415014153ab2e15291264e20af5a7f38db7adc6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 21 Jun 2018 14:05:17 +0300 Subject: [PATCH 141/239] use multiple jobs in acq --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 919526e4a..57429af7c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -371,7 +371,7 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30) + opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, acq_optimizer_kwargs={'n_jobs': -1}) with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From 8fee2e2409b87c6ee2964452439b3c431fd58019 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 21 Jun 2018 14:59:36 +0300 Subject: [PATCH 142/239] move result logging out from optimizer --- freqtrade/optimize/hyperopt.py | 72 ++++++++++++---------------------- 1 file changed, 25 insertions(+), 47 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 57429af7c..15c1a5e48 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -50,7 +50,6 @@ class Hyperopt(Backtesting): # to the number of days self.target_trades = 600 self.total_tries = config.get('epochs', 0) - self.current_tries = 0 self.current_best_loss = 100 # max average trade duration in minutes @@ -288,27 +287,8 @@ class Hyperopt(Backtesting): trade_count = len(results.index) trade_duration = results.trade_duration.mean() - if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: - print('.', end='') - sys.stdout.flush() - return { - 'status': STATUS_FAIL, - 'loss': float('inf') - } - loss = self.calculate_loss(total_profit, trade_count, trade_duration) - self.current_tries += 1 - - self.log_results( - { - 'loss': loss, - 'current_tries': self.current_tries, - 'total_tries': self.total_tries, - 'result': result_explanation, - } - ) - return { 'loss': loss, 'status': STATUS_OK, @@ -357,36 +337,34 @@ class Hyperopt(Backtesting): self.total_tries ) - try: - # best_parameters = fmin( - # fn=self.generate_optimizer, - # space=self.hyperopt_space(), - # algo=tpe.suggest, - # max_evals=self.total_tries, - # trials=self.trials - # ) + # results = sorted(self.trials.results, key=itemgetter('loss')) + # best_result = results[0]['result'] + cpus = multiprocessing.cpu_count() + print(f'Found {cpus}. Let\'s make them scream!') - # results = sorted(self.trials.results, key=itemgetter('loss')) - # best_result = results[0]['result'] - cpus = multiprocessing.cpu_count() - print(f'Found {cpus}. Let\'s make them scream!') + opt = Optimizer( + self.hyperopt_space(), + base_estimator="ET", + acq_optimizer="auto", + n_initial_points=30, + acq_optimizer_kwargs={'n_jobs': -1} + ) - opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, acq_optimizer_kwargs={'n_jobs': -1}) + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) - with Parallel(n_jobs=-1) as parallel: - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - #asked = opt.ask() - #f_val = self.generate_optimizer(asked) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, [i['loss'] for i in f_val]) - print(f'got value {f_val}') - - - except ValueError: - best_parameters = {} - best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ - 'try with more epochs (param: -e).' + for j in range(cpus): + self.log_results( + { + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + } + ) # Improve best parameter logging display # if best_parameters: From 8272120c3a4dae84f8935b22435c4721bdac0dd5 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 07:10:37 +0300 Subject: [PATCH 143/239] convert stoploss and ROI search spaces to skopt format --- freqtrade/optimize/hyperopt.py | 51 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 15c1a5e48..35fed3b7e 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -16,14 +16,14 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, Optional +from typing import Dict, Any, Callable, Optional, List import numpy import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame -from skopt.space import Real, Integer, Categorical +from skopt.space import Real, Integer, Categorical, Dimension from skopt import Optimizer from sklearn.externals.joblib import Parallel, delayed @@ -164,27 +164,27 @@ class Hyperopt(Backtesting): return roi_table @staticmethod - def roi_space() -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Values to search for each ROI steps """ - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] @staticmethod - def stoploss_space() -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ - Stoploss Value to search + Stoploss search space """ - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } + return [ + Real(-0.5, -0.02, name='stoploss'), + ] @staticmethod def indicator_space() -> Dict[str, Any]: @@ -211,19 +211,18 @@ class Hyperopt(Backtesting): return True return False - def hyperopt_space(self) -> Dict[str, Any]: + def hyperopt_space(self) -> List[Dimension]: """ Return the space to use during Hyperopt """ - return Hyperopt.indicator_space() - # spaces: Dict = {} - # if self.has_space('buy'): - # spaces = {**spaces, **Hyperopt.indicator_space()} - # if self.has_space('roi'): - # spaces = {**spaces, **Hyperopt.roi_space()} - # if self.has_space('stoploss'): - # spaces = {**spaces, **Hyperopt.stoploss_space()} - # return spaces + spaces: List[Dimension] = [] + if self.has_space('buy'): + spaces += Hyperopt.indicator_space() + if self.has_space('roi'): + spaces += Hyperopt.roi_space() + if self.has_space('stoploss'): + spaces += Hyperopt.stoploss_space() + return spaces @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: From a525cba8e9c94771606696788c9d50b29ee2f51f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 13:02:26 +0300 Subject: [PATCH 144/239] switch signal handler to try catch. fix pickling and formatting output --- freqtrade/optimize/hyperopt.py | 102 ++++++++++++--------------------- 1 file changed, 38 insertions(+), 64 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 35fed3b7e..960812248 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -8,7 +8,6 @@ import json import logging import os import pickle -import signal import sys import multiprocessing @@ -18,9 +17,7 @@ from math import exp from operator import itemgetter from typing import Dict, Any, Callable, Optional, List -import numpy import talib.abstract as ta -from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame from skopt.space import Real, Integer, Categorical, Dimension @@ -64,9 +61,9 @@ class Hyperopt(Backtesting): # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None - # Hyperopt Trials + # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = Trials() + self.trials = [] def get_args(self, params): dimensions = self.hyperopt_space() @@ -104,10 +101,11 @@ class Hyperopt(Backtesting): """ Save hyperopt trials to file """ - logger.info('Saving Trials to \'%s\'', self.trials_file) - pickle.dump(self.trials, open(self.trials_file, 'wb')) + if self.trials: + logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) - def read_trials(self) -> Trials: + def read_trials(self) -> List: """ Read hyperopt trials file """ @@ -120,9 +118,15 @@ class Hyperopt(Backtesting): """ Display Best hyperopt result """ - vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) - results = self.trials.best_trial['result']['result'] - logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + results = sorted(self.trials, key=itemgetter('loss')) + best_result = results[0] + logger.info( + 'Best result:\n%s\nwith values:\n%s', + best_result['result'], + best_result['params'] + ) + if 'roi_t1' in best_result['params']: + logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -202,7 +206,6 @@ class Hyperopt(Backtesting): Categorical([True, False], name='rsi-enabled'), ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -251,7 +254,7 @@ class Hyperopt(Backtesting): } #conditions.append(triggers.get(params['trigger']['type'])) - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -290,7 +293,7 @@ class Hyperopt(Backtesting): return { 'loss': loss, - 'status': STATUS_OK, + 'params': params, 'result': result_explanation, } @@ -322,22 +325,15 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, self.signal_handler) + logger.info('Preparing..') # read trials file if we have one if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: self.trials = self.read_trials() - - self.current_tries = len(self.trials.results) - self.total_tries += self.current_tries logger.info( - 'Continuing with trials. Current: %d, Total: %d', - self.current_tries, - self.total_tries + 'Loaded %d previous evaluations from disk.', + len(self.trials) ) - # results = sorted(self.trials.results, key=itemgetter('loss')) - # best_result = results[0]['result'] cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') @@ -349,50 +345,28 @@ class Hyperopt(Backtesting): acq_optimizer_kwargs={'n_jobs': -1} ) - with Parallel(n_jobs=-1) as parallel: - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, [i['loss'] for i in f_val]) + try: + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) - for j in range(cpus): - self.log_results( - { - 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, - 'total_tries': self.total_tries, - 'result': f_val[j]['result'], - } - ) - - # Improve best parameter logging display - # if best_parameters: - # best_parameters = space_eval( - # self.hyperopt_space(), - # best_parameters - # ) - - # logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - # if 'roi_t1' in best_parameters: - # logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) - - # logger.info('Best Result:\n%s', best_result) - - # # Store trials result to file to resume next time - # self.save_trials() - - def signal_handler(self, sig, frame) -> None: - """ - Hyperopt SIGINT handler - """ - logger.info( - 'Hyperopt received %s', - signal.Signals(sig).name - ) + self.trials += f_val + for j in range(cpus): + self.log_results( + { + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + } + ) + except KeyboardInterrupt: + print('User interrupted..') self.save_trials() self.log_trials_result() - sys.exit(0) def start(args: Namespace) -> None: From dde7df7fd31513d8cbb6b70527910d85fb46721c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 17:08:22 +0300 Subject: [PATCH 145/239] add scikit-optimize to dependencies --- requirements.txt | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index e26c9c8f6..5727e70c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,8 @@ networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 +# Required for hyperopt +scikit-optimize=0.5.2 + # Required for plotting data #plotly==2.7.0 diff --git a/setup.py b/setup.py index ee6b7ae38..cd0574fa2 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup(name='freqtrade', 'tabulate', 'cachetools', 'coinmarketcap', + 'scikit-optimize', ], include_package_data=True, zip_safe=False, From e8f2e6956d71026224d28ebf6ee37445764b998a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:02:59 +0300 Subject: [PATCH 146/239] to avoid pickle problems, get rid of reference to exchange after initialization --- freqtrade/optimize/hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 960812248..e53a00e4f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -325,6 +325,8 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) + self.exchange = None + logger.info('Preparing..') # read trials file if we have one if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: From 295dfe26523ba0cd959bb953e95c620650d31a63 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 13:50:22 +0200 Subject: [PATCH 147/239] persistence: remove obsolete global _CONF variable --- freqtrade/persistence.py | 7 ++----- freqtrade/tests/test_persistence.py | 11 ++--------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 7fd8fdeb9..20a7db4bb 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -21,7 +21,6 @@ from freqtrade import OperationalException logger = logging.getLogger(__name__) -_CONF = {} _DECL_BASE: Any = declarative_base() @@ -33,9 +32,7 @@ def init(config: Dict) -> None: :param config: config to use :return: None """ - _CONF.update(config) - - db_url = _CONF.get('db_url', None) + db_url = config.get('db_url', None) kwargs = {} # Take care of thread ownership if in-memory db @@ -61,7 +58,7 @@ def init(config: Dict) -> None: check_migrate(engine) # Clean dry_run DB if the db is not in-memory - if _CONF.get('dry_run', False) and db_url != 'sqlite://': + if config.get('dry_run', False) and db_url != 'sqlite://': clean_dry_run_db() diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index c50ad7d2c..30ad239a1 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -14,9 +14,7 @@ def init_persistence(default_conf): init(default_conf) -def test_init_create_session(default_conf, mocker): - mocker.patch.dict('freqtrade.persistence._CONF', default_conf) - +def test_init_create_session(default_conf): # Check if init create a session init(default_conf) assert hasattr(Trade, 'session') @@ -29,20 +27,17 @@ def test_init_custom_db_url(default_conf, mocker): # Update path to a value other than default, but still in-memory conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) - mocker.patch.dict('freqtrade.persistence._CONF', conf) init(conf) assert create_engine_mock.call_count == 1 assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite' -def test_init_invalid_db_url(default_conf, mocker): +def test_init_invalid_db_url(default_conf): conf = deepcopy(default_conf) # Update path to a value other than default, but still in-memory conf.update({'db_url': 'unknown:///some.url'}) - mocker.patch.dict('freqtrade.persistence._CONF', conf) - with pytest.raises(OperationalException, match=r'.*no valid database URL*'): init(conf) @@ -53,7 +48,6 @@ def test_init_prod_db(default_conf, mocker): conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) - mocker.patch.dict('freqtrade.persistence._CONF', conf) init(conf) assert create_engine_mock.call_count == 1 @@ -66,7 +60,6 @@ def test_init_dryrun_db(default_conf, mocker): conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) - mocker.patch.dict('freqtrade.persistence._CONF', conf) init(conf) assert create_engine_mock.call_count == 1 From 0b3e4f6bcd07d416f5ef3508ebfc76fd4cdfb2c5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 13:50:49 +0200 Subject: [PATCH 148/239] remove dead code --- freqtrade/tests/exchange/test_exchange.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 3c68e8493..620113be1 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -510,7 +510,6 @@ def test_cancel_order_dry_run(default_conf, mocker): # Ensure that if not dry_run, we should call API def test_cancel_order(default_conf, mocker): default_conf['dry_run'] = False - # mocker.patch.dict('freqtrade.exchange.._CONF', default_conf) api_mock = MagicMock() api_mock.cancel_order = MagicMock(return_value=123) exchange = get_patched_exchange(mocker, default_conf, api_mock) From 0440a1917169eb9ed2e6fde713903fe9baec311a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 14:19:50 +0200 Subject: [PATCH 149/239] export open/close rate for backtesting too preparation to allow plotting of backtest results --- freqtrade/optimize/backtesting.py | 22 +++++++++++++------- freqtrade/tests/optimize/test_backtesting.py | 18 +++++++++++----- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffb808a24..b5ed9b167 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -37,6 +37,8 @@ class BacktestResult(NamedTuple): close_index: int trade_duration: float open_at_end: bool + open_rate: float + close_rate: float class Backtesting(object): @@ -115,11 +117,13 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - records = [(trade_entry.pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - trade_entry.open_index - 1, trade_entry.trade_duration) - for index, trade_entry in results.iterrows()] + # columns = ["pair", "profit", "opents", "closets", "index", "duration", + # "open_rate", "close_rate", "open_at_end"] + + records = [(t.pair, t.profit_percent, t.open_time.timestamp(), + t.close_time.timestamp(), t.open_index - 1, t.trade_duration, + t.open_rate, t.close_rate, t.open_at_end) + for index, t in results.iterrows()] if records: logger.info('Dumping backtest results to %s', recordfilename) @@ -158,7 +162,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=False + open_at_end=False, + open_rate=buy_row.close, + close_rate=sell_row.close ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -171,7 +177,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=True + open_at_end=True, + open_rate=buy_row.close, + close_rate=sell_row.close ) logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 65aa00a70..4dda82463 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -604,9 +604,13 @@ def test_backtest_record(default_conf, fee, mocker): Arrow(2017, 11, 14, 22, 10, 00).datetime, Arrow(2017, 11, 14, 22, 43, 00).datetime, Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "open_index": [1, 119, 153, 185], "close_index": [118, 151, 184, 199], - "trade_duration": [123, 34, 31, 14]}) + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True] + }) backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 # Assert file_dump_json was only called once @@ -617,12 +621,16 @@ def test_backtest_record(default_conf, fee, mocker): # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None - for (pair, profit, date_buy, date_sell, buy_index, dur) in records: + for (pair, profit, date_buy, date_sell, buy_index, dur, + openr, closer, open_at_end) in records: assert pair == 'UNITTEST/BTC' - isinstance(profit, float) + assert isinstance(profit, float) # FIX: buy/sell should be converted to ints - isinstance(date_buy, str) - isinstance(date_sell, str) + assert isinstance(date_buy, float) + assert isinstance(date_sell, float) + assert isinstance(openr, float) + assert isinstance(closer, float) + assert isinstance(open_at_end, bool) isinstance(buy_index, pd._libs.tslib.Timestamp) if oix: assert buy_index > oix From 09261b11afa20af557a2b638bb952282a09dd5ac Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:22:14 +0300 Subject: [PATCH 150/239] remove hyperopt and networkx from dependencies --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5727e70c9..5754bf925 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,9 +15,6 @@ TA-Lib==0.4.17 pytest==3.6.2 pytest-mock==1.10.0 pytest-cov==2.5.1 -hyperopt==0.1 -# do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 -networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 From 925b9b0c1920c86c1e0e5df0abcb93602673b824 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 23 Jun 2018 14:23:07 +0200 Subject: [PATCH 151/239] Update ccxt from 1.14.253 to 1.14.256 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e26c9c8f6..51312791e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.253 +ccxt==1.14.256 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 9c66c258909df4b7673300cec43477d54a5dee4d Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 14:34:36 +0200 Subject: [PATCH 152/239] resolver: use current folder instead of script folder to find user_data --- freqtrade/strategy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3c7836291..73abb7cee 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -72,7 +72,7 @@ class StrategyResolver(object): """ current_path = os.path.dirname(os.path.realpath(__file__)) abs_paths = [ - os.path.join(current_path, '..', '..', 'user_data', 'strategies'), + os.path.join(os.getcwd(), 'user_data', 'strategies'), current_path, ] From 4ea5fcc6613b229345e576098dc0dc6a133a4029 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 14:42:22 +0200 Subject: [PATCH 153/239] resolver: don't fail if user_data can't be found --- freqtrade/strategy/resolver.py | 11 +++++++---- freqtrade/tests/strategy/test_strategy.py | 13 ++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 73abb7cee..0dcd3fc6a 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -81,10 +81,13 @@ class StrategyResolver(object): abs_paths.insert(0, extra_dir) for path in abs_paths: - strategy = self._search_strategy(path, strategy_name) - if strategy: - logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return import_strategy(strategy) + try: + strategy = self._search_strategy(path, strategy_name) + if strategy: + logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) + return import_strategy(strategy) + except FileNotFoundError: + logger.warning('Path "%s" does not exist', path) raise ImportError( "Impossible to load Strategy '{}'. This class does not exist" diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 13eac3261..1e082c380 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -50,13 +50,16 @@ def test_load_strategy(result): assert 'adx' in resolver.strategy.populate_indicators(result) -def test_load_strategy_custom_directory(result): +def test_load_strategy_invalid_directory(result, caplog): resolver = StrategyResolver() extra_dir = os.path.join('some', 'path') - with pytest.raises( - FileNotFoundError, - match=r".*No such file or directory: '{}'".format(extra_dir)): - resolver._load_strategy('TestStrategy', extra_dir) + resolver._load_strategy('TestStrategy', extra_dir) + + assert ( + 'freqtrade.strategy.resolver', + logging.WARNING, + 'Path "{}" does not exist'.format(extra_dir), + ) in caplog.record_tuples assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) From 136456afc0d657b2a9fe6e732f999ffe97ed5622 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:44:51 +0300 Subject: [PATCH 154/239] add three triggers to hyperopting --- freqtrade/optimize/hyperopt.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e53a00e4f..ea92cd561 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -94,6 +94,7 @@ class Hyperopt(Backtesting): # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) return dataframe @@ -204,6 +205,7 @@ class Hyperopt(Backtesting): Categorical([True, False], name='fastd-enabled'), Categorical([True, False], name='adx-enabled'), Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') ] def has_space(self, space: str) -> bool: @@ -238,8 +240,6 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS -# if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: -# conditions.append(dataframe['macd'] < 0) if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) if 'fastd' in params and params['fastd-enabled']: @@ -250,11 +250,16 @@ class Hyperopt(Backtesting): conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - triggers = { - } - #conditions.append(triggers.get(params['trigger']['type'])) - - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), From ab9e2fcea08692aa30abe5f20a48082c3452d34c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:47:19 +0300 Subject: [PATCH 155/239] fix guard names to match search space --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ea92cd561..f59b884b8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -242,11 +242,11 @@ class Hyperopt(Backtesting): # GUARDS AND TRENDS if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd' in params and params['fastd-enabled']: + if 'fastd-enabled' in params and params['fastd-enabled']: conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx' in params and params['adx-enabled']: + if 'adx-enabled' in params and params['adx-enabled']: conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi' in params and params['rsi-enabled']: + if 'rsi-enabled' in params and params['rsi-enabled']: conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS From 642ad02316aaa2a934545c9fb0841aa8ee7d5e89 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:56:38 +0300 Subject: [PATCH 156/239] remove unused import --- freqtrade/optimize/hyperopt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f59b884b8..f91647f6e 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,7 +4,6 @@ This module contains the hyperopt logic """ -import json import logging import os import pickle From 9bad75f37daccbc32ba43dfa45ab58cedf8e1a74 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 08:36:32 -0500 Subject: [PATCH 157/239] README: note to open an issue before starting major feature work --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 94ebb7bf8..929d40292 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,8 @@ Please read our [Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) to understand the requirements before sending your pull-requests. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. + **Important:** Always create your PR against the `develop` branch, not `master`. From 3384679bad36a5b199abe39b54d3b5a080da2095 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 09:38:20 -0500 Subject: [PATCH 158/239] bump develop to 0.17.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 7cf0fa996..ac00264f0 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.17.0' +__version__ = '0.17.1' class DependencyException(BaseException): From 3cedace2f6fce431cd5df784b5bcf5ce499bf02d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:54:27 +0200 Subject: [PATCH 159/239] add plotting for backtested trades --- scripts/plot_dataframe.py | 49 +++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7f9641222..cd41f51b1 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -27,9 +27,12 @@ Example of usage: import logging import os import sys +import json +from pathlib import Path from argparse import Namespace from typing import Dict, List, Any +import pandas as pd import plotly.graph_objs as go from plotly import tools from plotly.offline import plot @@ -103,10 +106,42 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exit() # Get trades already made from the DB - trades: List[Trade] = [] + trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades = Trade.query.filter(Trade.pair.is_(pair)).all() + trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() + # columns = ["pair", "profit", "opents", "closets", "index", "duration"] + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in trades_], columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] @@ -126,7 +161,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) -def generate_graph(pair, trades, data, args) -> tools.make_subplots: +def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: """ Generate the graph from the data generated by Backtesting or from DB :param pair: Pair to Display on the graph @@ -187,8 +222,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) trade_buys = go.Scattergl( - x=[t.open_date.isoformat() for t in trades], - y=[t.open_rate for t in trades], + x=trades["opents"], + y=trades["open_rate"], mode='markers', name='trade_buy', marker=dict( @@ -199,8 +234,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) ) trade_sells = go.Scattergl( - x=[t.close_date.isoformat() for t in trades], - y=[t.close_rate for t in trades], + x=trades["closets"], + y=trades["close_rate"], mode='markers', name='trade_sell', marker=dict( From f506ebcd62bf26b522afdd658dddd1fa3371deac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:58:28 +0200 Subject: [PATCH 160/239] use Pathlib in the whole script --- scripts/plot_dataframe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index cd41f51b1..763b8d9e4 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -25,7 +25,6 @@ Example of usage: --indicators2 fastk,fastd """ import logging -import os import sys import json from pathlib import Path @@ -158,7 +157,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: args=args ) - plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) + plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: From 5055563458597316bee1fbfafee1d42ba5eff673 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:14:15 +0200 Subject: [PATCH 161/239] add --plot-limit --- scripts/plot_dataframe.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 763b8d9e4..ba4da444a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -104,19 +104,20 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if tickers == {}: exit() + if args.db_url and args.exportfilename: + logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() - # columns = ["pair", "profit", "opents", "closets", "index", "duration"] columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] trades = pd.DataFrame([(t.pair, t.calc_profit(), t.open_date, t.close_date, t.open_rate, t.close_rate, t.close_date.timestamp() - t.open_date.timestamp()) - for t in trades_], columns=columns) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) if args.exportfilename: file = Path(args.exportfilename) @@ -147,13 +148,15 @@ def plot_analyzed_dataframe(args: Namespace) -> None: dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - if len(dataframe.index) > 750: - logger.warning('Ticker contained more than 750 candles, clipping.') - + if len(dataframe.index) > args.plot_limit: + logger.warning('Ticker contained more than %s candles as defined ' + 'with --plot-limit, clipping.', args.plot_limit) + dataframe = dataframe.tail(args.plot_limit) + trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] fig = generate_graph( pair=pair, trades=trades, - data=dataframe.tail(750), + data=dataframe, args=args ) @@ -333,11 +336,17 @@ def plot_parse_args(args: List[str]) -> Namespace: default='macd', dest='indicators2', ) - + arguments.parser.add_argument( + '--plot-limit', + help='Specify tick limit for plotting - too high values cause huge files - ' + 'Default: %(default)s', + dest='plot_limit', + default=750, + type=str, + ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) - return arguments.parse_args() From d8cb63efdde8d118b4939dda1d4e7a796a6402a6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:19:07 +0200 Subject: [PATCH 162/239] extract load_trades --- scripts/plot_dataframe.py | 77 +++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ba4da444a..e7e38fcd3 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -39,7 +39,7 @@ from plotly.offline import plot import freqtrade.optimize as optimize from freqtrade import persistence from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade @@ -48,6 +48,45 @@ logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} +def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: + trades: pd.DataFrame = pd.DataFrame() + if args.db_url: + persistence.init(_CONF) + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) + return trades + + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -107,41 +146,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if args.db_url and args.exportfilename: logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(_CONF) - columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date, t.close_date, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp()) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], - columns=columns) - - if args.exportfilename: - file = Path(args.exportfilename) - # must align with columns in backtest.py - columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] - with file.open() as f: - data = json.load(f) - trades = pd.DataFrame(data, columns=columns) - trades = trades.loc[trades["pair"] == pair] - if timerange: - if timerange.starttype == 'date': - trades = trades.loc[trades["opents"] >= timerange.startts] - if timerange.stoptype == 'date': - trades = trades.loc[trades["opents"] <= timerange.stopts] - - trades['opents'] = pd.to_datetime(trades['opents'], - unit='s', - utc=True, - infer_datetime_format=True) - trades['closets'] = pd.to_datetime(trades['closets'], - unit='s', - utc=True, - infer_datetime_format=True) + trades = load_trades(args, pair, timerange) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] From 660ec6f443046f6a2f02a2ec23ccd2a4ad4f9658 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 13:43:27 +0200 Subject: [PATCH 163/239] fix parameter type --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e7e38fcd3..1cc6b818a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -347,7 +347,7 @@ def plot_parse_args(args: List[str]) -> Namespace: 'Default: %(default)s', dest='plot_limit', default=750, - type=str, + type=int, ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) From 5e7e977ffab0b4f60f31ade8087bb038ec7c5ca6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 24 Jun 2018 14:23:05 +0200 Subject: [PATCH 164/239] Update ccxt from 1.14.256 to 1.14.257 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 51312791e..437bb5533 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.256 +ccxt==1.14.257 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 118a43cbb898658e3cfed8bb415d82d644daf944 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 24 Jun 2018 15:27:53 +0300 Subject: [PATCH 165/239] fixing tests for hyperopt --- freqtrade/optimize/hyperopt.py | 38 +++-- freqtrade/tests/optimize/test_hyperopt.py | 197 +++++----------------- 2 files changed, 61 insertions(+), 174 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f91647f6e..115cce1f0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -315,6 +315,18 @@ class Hyperopt(Backtesting): results.trade_duration.mean(), ) + def get_optimizer(self) -> Optimizer: + return Optimizer( + self.hyperopt_space(), + base_estimator="ET", + acq_optimizer="auto", + n_initial_points=30, + acq_optimizer_kwargs={'n_jobs': -1} + ) + + def run_optimizer_parallel(self, parallel, asked) -> List: + return parallel(delayed(self.generate_optimizer)(v) for v in asked) + def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -341,33 +353,25 @@ class Hyperopt(Backtesting): ) cpus = multiprocessing.cpu_count() - print(f'Found {cpus}. Let\'s make them scream!') + logger.info(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer( - self.hyperopt_space(), - base_estimator="ET", - acq_optimizer="auto", - n_initial_points=30, - acq_optimizer_kwargs={'n_jobs': -1} - ) + opt = self.get_optimizer() try: with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): asked = opt.ask(n_points=cpus) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val for j in range(cpus): - self.log_results( - { - 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, - 'total_tries': self.total_tries, - 'result': f_val[j]['result'], - } - ) + self.log_results({ + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + }) except KeyboardInterrupt: print('User interrupted..') diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 8ad1932af..0f3e56916 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import os -import signal from copy import deepcopy from unittest.mock import MagicMock @@ -42,16 +41,7 @@ def create_trials(mocker) -> None: mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) - return mocker.Mock( - results=[ - { - 'loss': 1, - 'result': 'foo', - 'status': 'ok' - } - ], - best_trial={'misc': {'vals': {'adx': 999}}} - ) + return [{'loss': 1, 'result': 'foo', 'params': {}}] # Unit tests @@ -148,74 +138,7 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: - fmin_result = { - "macd_below_zero": 0, - "adx": 1, - "adx-value": 15.0, - "fastd": 1, - "fastd-value": 40.0, - "green_candle": 1, - "mfi": 0, - "over_sar": 0, - "rsi": 1, - "rsi-value": 37.0, - "trigger": 2, - "uptrend_long_ema": 1, - "uptrend_short_ema": 0, - "uptrend_sma": 0, - "stoploss": -0.1, - "roi_t1": 1, - "roi_t2": 2, - "roi_t3": 3, - "roi_p1": 1, - "roi_p2": 2, - "roi_p3": 3, - } - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = create_trials(mocker) - hyperopt.tickerdata_to_dataframe = MagicMock() - hyperopt.start() - - exists = [ - 'Best parameters:', - '"adx": {\n "enabled": true,\n "value": 15.0\n },', - '"fastd": {\n "enabled": true,\n "value": 40.0\n },', - '"green_candle": {\n "enabled": true\n },', - '"macd_below_zero": {\n "enabled": false\n },', - '"mfi": {\n "enabled": false\n },', - '"over_sar": {\n "enabled": false\n },', - '"roi_p1": 1.0,', - '"roi_p2": 2.0,', - '"roi_p3": 3.0,', - '"roi_t1": 1.0,', - '"roi_t2": 2.0,', - '"roi_t3": 3.0,', - '"rsi": {\n "enabled": true,\n "value": 37.0\n },', - '"stoploss": -0.1,', - '"trigger": {\n "type": "faststoch10"\n },', - '"uptrend_long_ema": {\n "enabled": true\n },', - '"uptrend_short_ema": {\n "enabled": false\n },', - '"uptrend_sma": {\n "enabled": false\n }', - 'ROI table:\n{0: 6.0, 3.0: 3.0, 5.0: 1.0, 6.0: 0}', - 'Best Result:\nfoo' - ] - for line in exists: - assert line in caplog.text - - +@pytest.mark.skip(reason="Test not implemented") def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) @@ -244,6 +167,7 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> assert line in caplog.text +@pytest.mark.skip(reason="Waits for fixing") def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, default_conf) -> None: trials = create_trials(mocker) @@ -286,17 +210,18 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: - create_trials(mocker) + trials = create_trials(mocker) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) hyperopt = _HYPEROPT mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) + _HYPEROPT.trials = trials hyperopt.save_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') assert log_has( - 'Saving Trials to \'{}\''.format(trials_file), + 'Saving 1 evaluations to \'{}\''.format(trials_file), caplog.record_tuples ) mock_dump.assert_called_once() @@ -333,12 +258,14 @@ def test_roi_table_generation(init_hyperopt) -> None: assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} -def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: - trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) +def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) + ) patch_exchange(mocker) - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) @@ -347,11 +274,12 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: conf.update({'spaces': 'all'}) hyperopt = Hyperopt(conf) - hyperopt.trials = trials hyperopt.tickerdata_to_dataframe = MagicMock() hyperopt.start() - mock_fmin.assert_called_once() + parallel.assert_called_once() + + assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text def test_format_results(init_hyperopt): @@ -384,20 +312,6 @@ def test_format_results(init_hyperopt): assert result.find('Total profit 1.00000000 EUR') -def test_signal_handler(mocker, init_hyperopt): - """ - Test Hyperopt.signal_handler() - """ - m = MagicMock() - mocker.patch('sys.exit', m) - mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.save_trials', m) - mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.log_trials_result', m) - - hyperopt = _HYPEROPT - hyperopt.signal_handler(signal.SIGTERM, None) - assert m.call_count == 3 - - def test_has_space(init_hyperopt): """ Test Hyperopt.has_space() method @@ -422,8 +336,8 @@ def test_populate_indicators(init_hyperopt) -> None: # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe - assert 'ao' in dataframe - assert 'cci' in dataframe + assert 'mfi' in dataframe + assert 'rsi' in dataframe def test_buy_strategy_generator(init_hyperopt) -> None: @@ -437,44 +351,15 @@ def test_buy_strategy_generator(init_hyperopt) -> None: populate_buy_trend = _HYPEROPT.buy_strategy_generator( { - 'uptrend_long_ema': { - 'enabled': True - }, - 'macd_below_zero': { - 'enabled': True - }, - 'uptrend_short_ema': { - 'enabled': True - }, - 'mfi': { - 'enabled': True, - 'value': 20 - }, - 'fastd': { - 'enabled': True, - 'value': 20 - }, - 'adx': { - 'enabled': True, - 'value': 20 - }, - 'rsi': { - 'enabled': True, - 'value': 20 - }, - 'over_sar': { - 'enabled': True, - }, - 'green_candle': { - 'enabled': True, - }, - 'uptrend_sma': { - 'enabled': True, - }, - - 'trigger': { - 'type': 'lower_bb' - } + 'adx-value': 20, + 'fastd-value': 20, + 'mfi-value': 20, + 'rsi-value': 20, + 'adx-enabled': True, + 'fastd-enabled': True, + 'mfi-enabled': True, + 'rsi-enabled': True, + 'trigger': 'bb_lower' } ) result = populate_buy_trend(dataframe) @@ -505,33 +390,31 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: patch_exchange(mocker) optimizer_param = { - 'adx': {'enabled': False}, - 'fastd': {'enabled': True, 'value': 35.0}, - 'green_candle': {'enabled': True}, - 'macd_below_zero': {'enabled': True}, - 'mfi': {'enabled': False}, - 'over_sar': {'enabled': False}, - 'roi_p1': 0.01, - 'roi_p2': 0.01, - 'roi_p3': 0.1, + 'adx-value': 0, + 'fastd-value': 35, + 'mfi-value': 0, + 'rsi-value': 0, + 'adx-enabled': False, + 'fastd-enabled': True, + 'mfi-enabled': False, + 'rsi-enabled': False, + 'trigger': 'macd_cross_signal', 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, - 'rsi': {'enabled': False}, + 'roi_p1': 0.01, + 'roi_p2': 0.01, + 'roi_p3': 0.1, 'stoploss': -0.4, - 'trigger': {'type': 'macd_cross_signal'}, - 'uptrend_long_ema': {'enabled': False}, - 'uptrend_short_ema': {'enabled': True}, - 'uptrend_sma': {'enabled': True} } response_expected = { 'loss': 1.9840569076926293, 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' '(0.0231Σ%). Avg duration 100.0 mins.', - 'status': 'ok' + 'params': optimizer_param } hyperopt = Hyperopt(conf) - generate_optimizer_value = hyperopt.generate_optimizer(optimizer_param) + generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected From e70cb963f7cc9f82336c2fa07875b97d31e74c94 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 17:00:00 +0200 Subject: [PATCH 166/239] document what to do with exported backtest results --- docs/backtesting.md | 28 ++++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 3 --- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 1efb46b43..172969ae2 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -70,6 +70,34 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +The exported trades can be read using the following code for manual analysis, or can be used by the plotting script `plot_dataframe.py` in the scripts folder. + +``` python +import json +from pathlib import Path +import pandas as pd + +filename=Path('user_data/backtest_data/backtest-result.json') + +with filename.open() as file: + data = json.load(file) + +columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] +df = pd.DataFrame(data, columns=columns) + +df['opents'] = pd.to_datetime(df['opents'], + unit='s', + utc=True, + infer_datetime_format=True + ) +df['closets'] = pd.to_datetime(df['closets'], + unit='s', + utc=True, + infer_datetime_format=True + ) +``` + #### Exporting trades to file specifying a custom filename ```bash diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b5ed9b167..70f6168a4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -117,9 +117,6 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - # columns = ["pair", "profit", "opents", "closets", "index", "duration", - # "open_rate", "close_rate", "open_at_end"] - records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, t.open_rate, t.close_rate, t.open_at_end) From 43f1a1d264765f106ad9f94b5fd1f3ffb70d20de Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 19:52:12 +0200 Subject: [PATCH 167/239] rework download_backtest script --- freqtrade/arguments.py | 7 +++++ scripts/download_backtest_data.py | 48 ++++++++++++++++--------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 31232f1ff..8ce7c546d 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -334,3 +334,10 @@ class Arguments(object): nargs='+', dest='timeframes', ) + + self.parser.add_argument( + '--erase', + help='Clean all existing data for the selected exchange/pairs/timeframes', + dest='erase', + action='store_true' + ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 2f76c1232..686098f94 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -3,11 +3,14 @@ """This script generate json data from bittrex""" import json import sys -import os +from pathlib import Path import arrow -from freqtrade import (arguments, misc) +from freqtrade import arguments +from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange +from freqtrade.optimize import download_backtesting_testdata + DEFAULT_DL_PATH = 'user_data/data' @@ -17,25 +20,27 @@ args = arguments.parse_args() timeframes = args.timeframes -dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) +dl_path = Path(DEFAULT_DL_PATH).joinpath(args.exchange) if args.export: - dl_path = args.export + dl_path = Path(args.export) -if not os.path.isdir(dl_path): +if not dl_path.is_dir(): sys.exit(f'Directory {dl_path} does not exist.') -pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json') -if not os.path.isfile(pairs_file): +pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') +if not pairs_file.exists(): sys.exit(f'No pairs file found with path {pairs_file}.') -with open(pairs_file) as file: +with pairs_file.open() as file: PAIRS = list(set(json.load(file))) PAIRS.sort() -since_time = None + +timerange = TimeRange() if args.days: - since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000 + time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") + timerange = arguments.parse_timerange(f'{time_since}-') print(f'About to download pairs: {PAIRS} to {dl_path}') @@ -59,21 +64,18 @@ for pair in PAIRS: print(f"skipping pair {pair}") continue for tick_interval in timeframes: - print(f'downloading pair {pair}, interval {tick_interval}') - - data = exchange.get_ticker_history(pair, tick_interval, since_ms=since_time) - if not data: - print('\tNo data was downloaded') - break - - print('\tData was downloaded for period %s - %s' % ( - arrow.get(data[0][0] / 1000).format(), - arrow.get(data[-1][0] / 1000).format())) - - # save data pair_print = pair.replace('/', '_') filename = f'{pair_print}-{tick_interval}.json' - misc.file_dump_json(os.path.join(dl_path, filename), data) + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + print(f'Deleting existing data for pair {pair}, interval {tick_interval}') + dl_file.unlink() + + print(f'downloading pair {pair}, interval {tick_interval}') + download_backtesting_testdata(str(dl_path), exchange=exchange, + pair=pair, + tick_interval=tick_interval, + timerange=timerange) if pairs_not_available: From 17ee7f8be51ddb64b8c4ad1de88fc7edf266c647 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:15:11 +0300 Subject: [PATCH 168/239] fix typo in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5754bf925..c22dcbdca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Required for hyperopt -scikit-optimize=0.5.2 +scikit-optimize==0.5.2 # Required for plotting data #plotly==2.7.0 From 0bddc58ec41db007c75de5b16d1589e66cc3131d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:38:14 +0300 Subject: [PATCH 169/239] extract loading previous results to a method --- freqtrade/optimize/hyperopt.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 115cce1f0..b8b53ab3c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -327,6 +327,15 @@ class Hyperopt(Backtesting): def run_optimizer_parallel(self, parallel, asked) -> List: return parallel(delayed(self.generate_optimizer)(v) for v in asked) + def load_previous_results(self): + """ read trials file if we have one """ + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + logger.info( + 'Loaded %d previous evaluations from disk.', + len(self.trials) + ) + def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -340,23 +349,13 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - self.exchange = None - - logger.info('Preparing..') - # read trials file if we have one - if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: - self.trials = self.read_trials() - logger.info( - 'Loaded %d previous evaluations from disk.', - len(self.trials) - ) + self.load_previous_results() cpus = multiprocessing.cpu_count() logger.info(f'Found {cpus}. Let\'s make them scream!') opt = self.get_optimizer() - try: with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From 2b6407e5982dbeb8bd1038945507c7875592bfd4 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:38:42 +0300 Subject: [PATCH 170/239] remove unused tests from hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 72 ----------------------- 1 file changed, 72 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 0f3e56916..9194a66c9 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -44,7 +44,6 @@ def create_trials(mocker) -> None: return [{'loss': 1, 'result': 'foo', 'params': {}}] -# Unit tests def test_start(mocker, default_conf, caplog) -> None: """ Test start() function @@ -138,77 +137,6 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: assert caplog.record_tuples == [] -@pytest.mark.skip(reason="Test not implemented") -def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> None: - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = create_trials(mocker) - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - - exists = [ - 'Best Result:', - 'Sorry, Hyperopt was not able to find good parameters. Please try with more epochs ' - '(param: -e).', - ] - - for line in exists: - assert line in caplog.text - - -@pytest.mark.skip(reason="Waits for fixing") -def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, default_conf) -> None: - trials = create_trials(mocker) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results)) - mock_read = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.read_trials', - return_value=trials - ) - mock_save = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.save_trials', - return_value=None - ) - mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = trials - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - - mock_read.assert_called_once() - mock_save.assert_called_once() - - current_tries = hyperopt.current_tries - total_tries = hyperopt.total_tries - - assert current_tries == len(trials.results) - assert total_tries == (current_tries + len(trials.results)) - - def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) From 4f1fa28658d8c5d1ffa556f69022d2a56f8fdda8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 25 Jun 2018 14:23:06 +0200 Subject: [PATCH 171/239] Update ccxt from 1.14.257 to 1.14.267 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 437bb5533..c9d3a6529 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.257 +ccxt==1.14.267 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 7c2a50cef9b51edb4b5ef9e4ddb442096fed3a17 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 26 Jun 2018 14:23:06 +0200 Subject: [PATCH 172/239] Update ccxt from 1.14.267 to 1.14.272 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c9d3a6529..d4f8b2f06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.267 +ccxt==1.14.272 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 54f52fb36664c375ab8b2c4ce64ea08c40cc7e48 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Fri, 1 Jun 2018 20:50:40 -0700 Subject: [PATCH 173/239] Create stoploss.md --- docs/stoploss.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/stoploss.md diff --git a/docs/stoploss.md b/docs/stoploss.md new file mode 100644 index 000000000..ff93fd1d8 --- /dev/null +++ b/docs/stoploss.md @@ -0,0 +1,50 @@ +# Stop Loss support + +at this stage the bot contains the following stoploss support modes: + +1. static stop loss, defined in either the strategy or configuration + +2. trailing stop loss, defined in the configuration + +3. trailing stop loss, custom positive loss, defined in configuration + +## Static Stop Loss + +This is very simple, basically you define a stop loss of x in your strategy file or alternative in the configuration, which +will overwrite the strategy definition. This will basically try to sell your asset, the second the loss exceeds the defined loss. + +## Trail Stop Loss + +The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. +To enable this Feauture all you have to do, is to define the configuration element: + +``` +"trailing_stop" : True +``` +This will now actiave an algorithm, whihch automatically moves up your stop loss, every time the price of your asset increases. + +For example, simplified math, + +* you buy an asset at a price of 100$ +* your stop loss is defined at 2% +* which means your stop loss, gets triggered once your asset dropped below 98$ +* assuming your asset now increases in proce to 102$ +* your stop loss, will now be 2% of 102$ or 99.96$ +* now your asset drops in value to 101$, your stop loss, will still be 99.96$ + +basically what this means, is that your stop loss will be adjusted to be always be 2% of the highest observed price + +### Custom positive loss + +due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, +the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the +black, it will be changed to be only a 1% stop loss + +this can be configured in the main confiuration file, the following way: + +``` + "trailing_stop": { + "positive" : 0.01 + }, +``` +The 0.01 would translate to a 1% stop loss, once you hit profit. From 257e1847b121dce297f5dc540ca52e3656c74517 Mon Sep 17 00:00:00 2001 From: peterkorodi Date: Sun, 17 Jun 2018 11:00:34 +0200 Subject: [PATCH 174/239] Update stoploss.md --- docs/stoploss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index ff93fd1d8..110cfa0d1 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -40,7 +40,7 @@ due to demand, it is possible to have a default stop loss, when you are in the r the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss -this can be configured in the main confiuration file, the following way: +this can be configured in the main configuration file, the following way: ``` "trailing_stop": { From 9ac3c559b6028bf0b382ed4a8adc8e0786f6b57c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 20:26:28 +0200 Subject: [PATCH 175/239] fix some stoploss documentation --- docs/stoploss.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 110cfa0d1..cf6a2bcab 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -1,11 +1,9 @@ # Stop Loss support -at this stage the bot contains the following stoploss support modes: +At this stage the bot contains the following stoploss support modes: 1. static stop loss, defined in either the strategy or configuration - 2. trailing stop loss, defined in the configuration - 3. trailing stop loss, custom positive loss, defined in configuration ## Static Stop Loss @@ -16,35 +14,37 @@ will overwrite the strategy definition. This will basically try to sell your ass ## Trail Stop Loss The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. -To enable this Feauture all you have to do, is to define the configuration element: +To enable this Feauture all you have to do is to define the configuration element: -``` +``` json "trailing_stop" : True ``` -This will now actiave an algorithm, whihch automatically moves up your stop loss, every time the price of your asset increases. + +This will now activate an algorithm, which automatically moves your stop loss up every time the price of your asset increases. For example, simplified math, * you buy an asset at a price of 100$ * your stop loss is defined at 2% * which means your stop loss, gets triggered once your asset dropped below 98$ -* assuming your asset now increases in proce to 102$ +* assuming your asset now increases to 102$ * your stop loss, will now be 2% of 102$ or 99.96$ * now your asset drops in value to 101$, your stop loss, will still be 99.96$ -basically what this means, is that your stop loss will be adjusted to be always be 2% of the highest observed price +basically what this means is that your stop loss will be adjusted to be always be 2% of the highest observed price ### Custom positive loss -due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, +Due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss this can be configured in the main configuration file, the following way: -``` +``` json "trailing_stop": { "positive" : 0.01 }, ``` + The 0.01 would translate to a 1% stop loss, once you hit profit. From 243c36b39bda5e126cd9ca6f805e70b15a57deab Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 20:49:07 +0200 Subject: [PATCH 176/239] get persistence.py for stop_loss --- freqtrade/persistence.py | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 20a7db4bb..76a499eed 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -151,6 +151,12 @@ class Trade(_DECL_BASE): open_date = Column(DateTime, nullable=False, default=datetime.utcnow) close_date = Column(DateTime) open_order_id = Column(String) + # absolute value of the stop loss + stop_loss = Column(Float, nullable=True, default=0.0) + # absolute value of the initial stop loss + initial_stop_loss = Column(Float, nullable=True, default=0.0) + # absolute value of the highest reached price + max_rate = Column(Float, nullable=True, default=0.0) def __repr__(self): return 'Trade(id={}, pair={}, amount={:.8f}, open_rate={:.8f}, open_since={})'.format( @@ -161,6 +167,49 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) + def adjust_stop_loss(self, current_price, stoploss): + """ + this adjusts the stop loss to it's most recently observed + setting + :param current_price: + :param stoploss: + :return: + """ + + new_loss = Decimal(current_price * (1 - abs(stoploss))) + + # keeping track of the highest observed rate for this trade + if self.max_rate is None: + self.max_rate = current_price + else: + if current_price > self.max_rate: + self.max_rate = current_price + + # no stop loss assigned yet + if self.stop_loss is None or self.stop_loss == 0: + logger.debug("assigning new stop loss") + self.stop_loss = new_loss + self.initial_stop_loss = new_loss + + # evaluate if the stop loss needs to be updated + else: + if new_loss > self.stop_loss: # stop losses only walk up, never down! + self.stop_loss = new_loss + logger.debug("adjusted stop loss") + else: + logger.debug("keeping current stop loss") + + logger.debug( + "{} - current price {:.8f}, bought at {:.8f} and calculated " + "stop loss is at: {:.8f} initial stop at {:.8f}. trailing stop loss saved us: {:.8f} " + "and max observed rate was {:.8f}".format( + self.pair, current_price, self.open_rate, + self.initial_stop_loss, + self.stop_loss, float(self.stop_loss) - float(self.initial_stop_loss), + self.max_rate + + )) + def update(self, order: Dict) -> None: """ Updates this entity with amount and actual open/close rates. From 5015bc9bb09d8904131e37d7d26addd47a025e3a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 22:41:28 +0200 Subject: [PATCH 177/239] slight update to persistence --- freqtrade/persistence.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 76a499eed..db97fc858 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -176,7 +176,7 @@ class Trade(_DECL_BASE): :return: """ - new_loss = Decimal(current_price * (1 - abs(stoploss))) + new_loss = float(current_price * (1 - abs(stoploss))) # keeping track of the highest observed rate for this trade if self.max_rate is None: @@ -200,15 +200,13 @@ class Trade(_DECL_BASE): logger.debug("keeping current stop loss") logger.debug( - "{} - current price {:.8f}, bought at {:.8f} and calculated " - "stop loss is at: {:.8f} initial stop at {:.8f}. trailing stop loss saved us: {:.8f} " - "and max observed rate was {:.8f}".format( - self.pair, current_price, self.open_rate, - self.initial_stop_loss, - self.stop_loss, float(self.stop_loss) - float(self.initial_stop_loss), - self.max_rate - - )) + f"{self.pair} - current price {current_price:.8f}, " + f"bought at {self.open_rate:.8f} and calculated " + f"stop loss is at: {self.initial_stop_loss:.8f} initial " + f"stop at {self.stop_loss:.8f}. " + f"trailing stop loss saved us: " + f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f} " + f"and max observed rate was {self.max_rate:.8f}") def update(self, order: Dict) -> None: """ From 3e167e11705b204bb2dcc504bf511bd7b2d9f6ca Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 22:41:38 +0200 Subject: [PATCH 178/239] update sample configs --- config.json.example | 5 ++++- config_full.json.example | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index e5bdc95b1..7b53ffa99 100644 --- a/config.json.example +++ b/config.json.example @@ -5,6 +5,9 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, + "trailing_stop": { + "positive" : 0.005 + }, "unfilledtimeout": 600, "bid_strategy": { "ask_last_balance": 0.0 @@ -32,7 +35,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false + "sell_fullfilled_at_roi": false }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index 231384acc..d7d8f7369 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -5,6 +5,7 @@ "fiat_display_currency": "USD", "dry_run": false, "ticker_interval": "5m", + "trailing_stop": true, "minimal_roi": { "40": 0.0, "30": 0.01, From da5be9fbd0ea132a9490ecca007d2e8adf568c34 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:06:27 +0200 Subject: [PATCH 179/239] add stop_loss based on work from @berlinguyinca --- config.json.example | 4 +--- config_full.json.example | 3 ++- docs/stoploss.md | 6 ++---- freqtrade/analyze.py | 42 ++++++++++++++++++++++++++++++++++++---- freqtrade/constants.py | 2 ++ 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/config.json.example b/config.json.example index 7b53ffa99..f92849ec9 100644 --- a/config.json.example +++ b/config.json.example @@ -5,9 +5,7 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, - "trailing_stop": { - "positive" : 0.005 - }, + "trailing_stop": false, "unfilledtimeout": 600, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/config_full.json.example b/config_full.json.example index d7d8f7369..244daabac 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -5,7 +5,8 @@ "fiat_display_currency": "USD", "dry_run": false, "ticker_interval": "5m", - "trailing_stop": true, + "trailing_stop": false, + "trailing_stop_positive": 0.005, "minimal_roi": { "40": 0.0, "30": 0.01, diff --git a/docs/stoploss.md b/docs/stoploss.md index cf6a2bcab..db4433a02 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -39,12 +39,10 @@ Due to demand, it is possible to have a default stop loss, when you are in the r the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss -this can be configured in the main configuration file, the following way: +This can be configured in the main configuration file and requires `"trailing_stop": true` to be set to true. ``` json - "trailing_stop": { - "positive" : 0.01 - }, + "trailing_stop_positive": 0.01, ``` The 0.01 would translate to a 1% stop loss, once you hit profit. diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 36e00dd0e..b283e3ae9 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -180,7 +180,7 @@ class Analyze(object): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - if self.stop_loss_reached(current_profit=current_profit): + if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date): return True experimental = self.config.get('experimental', {}) @@ -204,12 +204,46 @@ class Analyze(object): return False - def stop_loss_reached(self, current_profit: float) -> bool: - """Based on current profit of the trade and configured stoploss, decides to sell or not""" + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime) -> bool: + """ + Based on current profit of the trade and configured (trailing) stoploss, + decides to sell or not + """ + + current_profit = trade.calc_profit_percent(current_rate) + trailing_stop = self.config.get('trailing_stop', False) + if trade.stop_loss is None: + # initially adjust the stop loss to the base value + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) + + # evaluate if the stoploss was hit + if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: + + if trailing_stop: + logger.debug( + f"HIT STOP: current price at {current_rate:.6f}, " + f"stop loss is {trade.stop_loss:.6f}, " + f"initial stop loss was at {trade.initial_stop_loss:.6f}, " + f"trade opened at {trade.open_rate:.6f}") + logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: logger.debug('Stop loss hit.') return True + + # update the stop loss afterwards, after all by definition it's supposed to be hanging + if trailing_stop: + + # check if we have a special stop loss for positive condition + # and if profit is positive + stop_loss_value = self.strategy.stoploss + if 'trailing_stop_positive' in self.config and current_profit > 0: + + stop_loss_value = self.config.get('trailing_stop_positive') + logger.debug(f"using positive stop loss mode: {stop_loss_value} " + f"since we have profit {current_profit}") + + trade.adjust_stop_loss(current_rate, stop_loss_value) + return False def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..4ba1a7094 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -61,6 +61,8 @@ CONF_SCHEMA = { 'minProperties': 1 }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, + 'trailing_stop': {'type': 'boolean'}, + 'trailing_stop_positive': {'type': 'number', 'minimum': 0}, 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, 'bid_strategy': { 'type': 'object', From 03005bc0f14f9f7747652a13980c8f8a209ded11 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:14:12 +0200 Subject: [PATCH 180/239] update documentation --- docs/configuration.md | 42 ++++++++++++++++++++++++++++++++++-------- docs/index.md | 2 ++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5279ed06e..f24adb61e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,12 +1,15 @@ # Configure the bot + This page explains how to configure your `config.json` file. ## Table of Contents + - [Bot commands](#bot-commands) - [Backtesting commands](#backtesting-commands) - [Hyperopt commands](#hyperopt-commands) ## Setup config.json + We recommend to copy and use the `config.json.example` as a template for your bot configuration. @@ -22,6 +25,8 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. +| `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). +| `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. | `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). @@ -41,10 +46,10 @@ The table below will list all configuration parameters. | `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. -The definition of each config parameters is in -[misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). +The definition of each config parameters is in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). ### Understand stake_amount + `stake_amount` is an amount of crypto-currency your bot will use for each trade. The minimal value is 0.0005. If there is not enough crypto-currency in the account an exception is generated. @@ -52,10 +57,12 @@ To allow the bot to trade all the avaliable `stake_currency` in your account set In this case a trade amount is calclulated as `currency_balanse / (max_open_trades - current_open_trades)`. ### Understand minimal_roi + `minimal_roi` is a JSON object where the key is a duration in minutes and the value is the minimum ROI in percent. See the example below: -``` + +``` json "minimal_roi": { "40": 0.0, # Sell after 40 minutes if the profit is not negative "30": 0.01, # Sell after 30 minutes if there is at least 1% profit @@ -69,6 +76,7 @@ value. This parameter is optional. If you use it, it will take over the `minimal_roi` value from the strategy file. ### Understand stoploss + `stoploss` is loss in percentage that should trigger a sale. For example value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. @@ -77,82 +85,100 @@ Most of the strategy files already include the optimal `stoploss` value. This parameter is optional. If you use it, it will take over the `stoploss` value from the strategy file. +### Understand trailing stoploss + +Go to the [trailing stoploss Documentation](stoploss.md) for details on trailing stoploss. + ### Understand initial_state + `initial_state` is an optional field that defines the initial application state. Possible values are `running` or `stopped`. (default=`running`) If the value is `stopped` the bot has to be started with `/start` first. ### Understand process_throttle_secs + `process_throttle_secs` is an optional field that defines in seconds how long the bot should wait before asking the strategy if we should buy or a sell an asset. After each wait period, the strategy is asked again for every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or the static list of pairs) if we should buy. ### Understand ask_last_balance + `ask_last_balance` sets the bidding price. Value `0.0` will use `ask` price, `1.0` will use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. ### What values for exchange.name? + Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested with only Bittrex and Binance. The bot was tested with the following exchanges: + - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" Feel free to test other exchanges and submit your PR to improve the bot. ### What values for fiat_display_currency? + `fiat_display_currency` set the base currency to use for the conversion from coin to fiat in Telegram. The valid values are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". In addition to central bank currencies, a range of cryto currencies are supported. The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT". ## Switch to dry-run mode + We recommend starting the bot in dry-run mode to see how your bot will behave and how is the performance of your strategy. In Dry-run mode the bot does not engage your money. It only runs a live simulation without creating trades. ### To switch your bot in Dry-run mode: + 1. Edit your `config.json` file 2. Switch dry-run to true and specify db_url for a persistent db + ```json "dry_run": true, "db_url": "sqlite///tradesv3.dryrun.sqlite", ``` 3. Remove your Exchange API key (change them by fake api credentials) + ```json "exchange": { "name": "bittrex", "key": "key", "secret": "secret", ... -} +} ``` Once you will be happy with your bot performance, you can switch it to production mode. ## Switch to production mode + In production mode, the bot will engage your money. Be careful a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. ### To switch your bot in production mode: + 1. Edit your `config.json` file 2. Switch dry-run to false and don't forget to adapt your database URL if set + ```json "dry_run": false, ``` 3. Insert your Exchange API key (change them by fake api keys) + ```json "exchange": { "name": "bittrex", @@ -160,10 +186,10 @@ you run it in production mode. "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5", ... } + ``` -If you have not your Bittrex API key yet, -[see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). +If you have not your Bittrex API key yet, [see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). ## Next step -Now you have configured your config.json, the next step is to -[start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). + +Now you have configured your config.json, the next step is to [start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). diff --git a/docs/index.md b/docs/index.md index afde2d5eb..fd6bf4378 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,5 @@ # freqtrade documentation + Welcome to freqtrade documentation. Please feel free to contribute to this documentation if you see it became outdated by sending us a Pull-request. Do not hesitate to reach us on @@ -6,6 +7,7 @@ Pull-request. Do not hesitate to reach us on if you do not find the answer to your questions. ## Table of Contents + - [Pre-requisite](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md) - [Setup your Bittrex account](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-bittrex-account) - [Setup your Telegram bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-telegram-bot) From a3708bc56e17912af630cd2258f2920ec9d4ec17 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:40:20 +0200 Subject: [PATCH 181/239] add missing test --- freqtrade/tests/test_analyze.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 89dac21c6..02225acca 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -42,6 +42,7 @@ def test_analyze_object() -> None: assert hasattr(Analyze, 'get_signal') assert hasattr(Analyze, 'should_sell') assert hasattr(Analyze, 'min_roi_reached') + assert hasattr(Analyze, 'stop_loss_reached') def test_dataframe_correct_length(result): From 8bec505bbe23919da0035f86627d63a150ab445d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:40:36 +0200 Subject: [PATCH 182/239] add test for trailing_stoploss --- freqtrade/tests/test_freqtradebot.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..3568dad1d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1654,6 +1654,43 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> assert freqtrade.handle_trade(trade) is True +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['trailing_stop'] = True + print(limit_buy_order) + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + trade.stop_loss = 3.2 + caplog.set_level(logging.DEBUG) + assert freqtrade.handle_trade(trade) is True + assert log_has( + f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' + f'initial stop loss was at 0.000000, trade opened at 0.000011', caplog.record_tuples) + + + def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ From 88b898cce4a434cd15567581f42663056fe05da0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 00:16:19 +0200 Subject: [PATCH 183/239] add test for moving stoploss --- freqtrade/tests/test_freqtradebot.py | 63 ++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 3568dad1d..468f5f8aa 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1661,14 +1661,14 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 + 'bid': 0.00000102, + 'ask': 0.00000103, + 'last': 0.00000102 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1682,14 +1682,61 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) trade = Trade.query.first() trade.update(limit_buy_order) - trade.stop_loss = 3.2 caplog.set_level(logging.DEBUG) + # Sell as trailing-stop is reached + assert freqtrade.handle_trade(trade) is True + assert log_has( + f'HIT STOP: current price at 0.000001, stop loss is {trade.stop_loss:.6f}, ' + f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + + +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.0000182, + 'ask': 0.0000183, + 'last': 0.0000182 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['trailing_stop'] = True + conf['trailing_stop_positive'] = 0.01 + print(limit_buy_order) + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + caplog.set_level(logging.DEBUG) + # stop-loss not reached + assert freqtrade.handle_trade(trade) is False + # Adjusted stoploss + assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.64779142', + caplog.record_tuples) + assert log_has(f'adjusted stop loss', caplog.record_tuples) + assert trade.stop_loss == 0.000018018 + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 + })) assert freqtrade.handle_trade(trade) is True assert log_has( f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' - f'initial stop loss was at 0.000000, trade opened at 0.000011', caplog.record_tuples) - - + f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: From e9d5bceeb98b12aa35c020fea29d84bf24088ed4 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 00:18:50 +0200 Subject: [PATCH 184/239] cleanly check if stop_loss is initialized --- freqtrade/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b283e3ae9..b8caa45b7 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -212,7 +212,7 @@ class Analyze(object): current_profit = trade.calc_profit_percent(current_rate) trailing_stop = self.config.get('trailing_stop', False) - if trade.stop_loss is None: + if trade.stop_loss is None or trade.stop_loss == 0: # initially adjust the stop loss to the base value trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) From a91d75b3b2acdcce30a5e7bc9745430072da00d3 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:23:49 +0200 Subject: [PATCH 185/239] Add test for adjust_stop-loss --- freqtrade/tests/test_persistence.py | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 30ad239a1..6ffcbbf65 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -453,3 +453,37 @@ def test_migrate_new(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "binance" + + +def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + ) + + # Get percent of profit with a custom rate (Higher than open rate) + trade.adjust_stop_loss(1, 0.05) + assert trade.stop_loss == 0.95 + assert trade.max_rate == 1 + assert trade.initial_stop_loss == 0.95 + + trade.adjust_stop_loss(1.3, -0.1) + assert round(trade.stop_loss, 8) == 1.17 + assert trade.max_rate == 1.3 + assert trade.initial_stop_loss == 0.95 + + # current rate lower ... should not change + trade.adjust_stop_loss(1.2, 0.1) + assert round(trade.stop_loss, 8) == 1.17 + assert trade.max_rate == 1.3 + assert trade.initial_stop_loss == 0.95 + + # current rate higher... should raise stoploss + trade.adjust_stop_loss(1.4, 0.1) + assert round(trade.stop_loss, 8) == 1.26 + assert trade.max_rate == 1.4 + assert trade.initial_stop_loss == 0.95 From c997aa9864ef0aec7b89bfb59982f7b99a609e39 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:38:49 +0200 Subject: [PATCH 186/239] move initial logic to persistence --- freqtrade/analyze.py | 5 ++--- freqtrade/persistence.py | 12 ++++++++---- freqtrade/tests/test_persistence.py | 18 +++++++++++++++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b8caa45b7..d74f9566a 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -212,9 +212,8 @@ class Analyze(object): current_profit = trade.calc_profit_percent(current_rate) trailing_stop = self.config.get('trailing_stop', False) - if trade.stop_loss is None or trade.stop_loss == 0: - # initially adjust the stop loss to the base value - trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) + + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) # evaluate if the stoploss was hit if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index db97fc858..4fc83a18f 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -167,15 +167,19 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) - def adjust_stop_loss(self, current_price, stoploss): + def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool =False): """ - this adjusts the stop loss to it's most recently observed - setting + this adjusts the stop loss to it's most recently observed setting :param current_price: - :param stoploss: + :param stoploss in percent: + :param initial: :return: """ + if initial and not (self.stop_loss is None or self.stop_loss == 0): + # Don't modify if called with initial and nothing to do + return + new_loss = float(current_price * (1 - abs(stoploss))) # keeping track of the highest observed rate for this trade diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 6ffcbbf65..45957853f 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -465,18 +465,24 @@ def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): open_rate=1, ) - # Get percent of profit with a custom rate (Higher than open rate) - trade.adjust_stop_loss(1, 0.05) + trade.adjust_stop_loss(trade.open_rate, 0.05, True) assert trade.stop_loss == 0.95 assert trade.max_rate == 1 assert trade.initial_stop_loss == 0.95 + # Get percent of profit with a lowre rate + trade.adjust_stop_loss(0.96, 0.05) + assert trade.stop_loss == 0.95 + assert trade.max_rate == 1 + assert trade.initial_stop_loss == 0.95 + + # Get percent of profit with a custom rate (Higher than open rate) trade.adjust_stop_loss(1.3, -0.1) assert round(trade.stop_loss, 8) == 1.17 assert trade.max_rate == 1.3 assert trade.initial_stop_loss == 0.95 - # current rate lower ... should not change + # current rate lower again ... should not change trade.adjust_stop_loss(1.2, 0.1) assert round(trade.stop_loss, 8) == 1.17 assert trade.max_rate == 1.3 @@ -487,3 +493,9 @@ def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): assert round(trade.stop_loss, 8) == 1.26 assert trade.max_rate == 1.4 assert trade.initial_stop_loss == 0.95 + + # Initial is true but stop_loss set - so doesn't do anything + trade.adjust_stop_loss(1.7, 0.1, True) + assert round(trade.stop_loss, 8) == 1.26 + assert trade.max_rate == 1.4 + assert trade.initial_stop_loss == 0.95 From 78e6c9fdf680307c262e1246d02e4d76a877c90a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:51:48 +0200 Subject: [PATCH 187/239] add tests for trailing stoploss --- freqtrade/tests/test_freqtradebot.py | 41 ++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 468f5f8aa..7b2786421 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1694,6 +1694,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, """ Test sell_profit_only feature when enabled and we have a loss """ + buy_price = limit_buy_order['price'] patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) @@ -1702,9 +1703,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.0000182, - 'ask': 0.0000183, - 'last': 0.0000182 + 'bid': buy_price - 0.000001, + 'ask': buy_price - 0.000001, + 'last': buy_price - 0.000001 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1713,7 +1714,6 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, conf = deepcopy(default_conf) conf['trailing_stop'] = True conf['trailing_stop_positive'] = 0.01 - print(limit_buy_order) freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -1722,22 +1722,35 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, caplog.set_level(logging.DEBUG) # stop-loss not reached assert freqtrade.handle_trade(trade) is False - # Adjusted stoploss - assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.64779142', - caplog.record_tuples) - assert log_has(f'adjusted stop loss', caplog.record_tuples) - assert trade.stop_loss == 0.000018018 + + # Raise ticker above buy price mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 - })) + 'bid': buy_price + 0.000003, + 'ask': buy_price + 0.000003, + 'last': buy_price + 0.000003 + })) + # stop-loss not reached, adjusted stoploss + assert freqtrade.handle_trade(trade) is False + assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.26662643', + caplog.record_tuples) + assert log_has(f'adjusted stop loss', caplog.record_tuples) + assert trade.stop_loss == 0.0000138501 + + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.000002, + 'ask': buy_price + 0.000002, + 'last': buy_price + 0.000002 + })) + # Lower price again (but still positive) assert freqtrade.handle_trade(trade) is True assert log_has( - f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' + f'HIT STOP: current price at {buy_price + 0.000002:.6f}, ' + f'stop loss is {trade.stop_loss:.6f}, ' f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ From e6e868a03c3f4b6f94d89449509b8c618f53b84f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:54:29 +0200 Subject: [PATCH 188/239] remove markdown code type as it is not valid json --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index f24adb61e..984f2529b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -62,7 +62,7 @@ In this case a trade amount is calclulated as `currency_balanse / (max_open_trad in minutes and the value is the minimum ROI in percent. See the example below: -``` json +``` "minimal_roi": { "40": 0.0, # Sell after 40 minutes if the profit is not negative "30": 0.01, # Sell after 30 minutes if there is at least 1% profit From 8ecdae67e1417de14177da25ce30df8a28c7ad66 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:57:41 +0200 Subject: [PATCH 189/239] add mypy ignore (and comment as to why) --- freqtrade/analyze.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index d74f9566a..bf6b89b17 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -237,7 +237,8 @@ class Analyze(object): stop_loss_value = self.strategy.stoploss if 'trailing_stop_positive' in self.config and current_profit > 0: - stop_loss_value = self.config.get('trailing_stop_positive') + # Ignore mypy error check in configuration that this is a float + stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore logger.debug(f"using positive stop loss mode: {stop_loss_value} " f"since we have profit {current_profit}") From 19beb0941f4354e00a89238ff1630d86d5df9050 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 27 Jun 2018 14:23:07 +0200 Subject: [PATCH 190/239] Update ccxt from 1.14.272 to 1.14.288 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4f8b2f06..62e2e3983 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.272 +ccxt==1.14.288 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 860b270e309a022e607d789754eb3ab338475d58 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 19:49:08 +0200 Subject: [PATCH 191/239] update db migrate script to work for more changes --- freqtrade/persistence.py | 33 +++++++++++++++++++---------- freqtrade/tests/test_persistence.py | 3 +++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 4fc83a18f..c72974c61 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -66,6 +66,10 @@ def has_column(columns, searchname: str) -> bool: return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 +def get_column_def(columns, column: str, default: str) -> str: + return default if not has_column(columns, column) else column + + def check_migrate(engine) -> None: """ Checks if migration is necessary and migrates if necessary @@ -74,17 +78,26 @@ def check_migrate(engine) -> None: cols = inspector.get_columns('trades') - if not has_column(cols, 'fee_open'): + # Check for latest column + if not has_column(cols, 'max_rate'): + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + max_rate = get_column_def(cols, 'max_rate', '0.0') + # Schema migration necessary engine.execute("alter table trades rename to trades_bak") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) # Copy data back - following the correct schema - engine.execute("""insert into trades + engine.execute(f"""insert into trades (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id) + stake_amount, amount, open_date, close_date, open_order_id, + stop_loss, initial_stop_loss, max_rate + ) select id, lower(exchange), case when instr(pair, '_') != 0 then @@ -94,9 +107,12 @@ def check_migrate(engine) -> None: end pair, is_open, fee fee_open, fee fee_close, - open_rate, null open_rate_requested, close_rate, - null close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, + {max_rate} max_rate + from trades_bak """) @@ -104,11 +120,6 @@ def check_migrate(engine) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') - if not has_column(cols, 'open_rate_requested'): - engine.execute("alter table trades add open_rate_requested float") - if not has_column(cols, 'close_rate_requested'): - engine.execute("alter table trades add close_rate_requested float") - def cleanup() -> None: """ diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 45957853f..0cd2d089c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -400,6 +400,9 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "bittrex" + assert trade.max_rate == 0.0 + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 def test_migrate_new(mocker, default_conf, fee): From d5ad066f8d4af30340da2e12eaef61a88bf48224 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 20:15:25 +0200 Subject: [PATCH 192/239] support multiple db transitions by keeping the backup-table dynamic --- freqtrade/persistence.py | 12 +++++++++--- freqtrade/tests/test_persistence.py | 10 +++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c72974c61..2b226e53a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -77,6 +77,13 @@ def check_migrate(engine) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') + tabs = inspector.get_table_names() + table_back_name = 'trades_bak' + i = 0 + while table_back_name in tabs: + i += 1 + table_back_name = f'trades_bak{i}' + logger.info(f'trying {table_back_name}') # Check for latest column if not has_column(cols, 'max_rate'): @@ -87,7 +94,7 @@ def check_migrate(engine) -> None: max_rate = get_column_def(cols, 'max_rate', '0.0') # Schema migration necessary - engine.execute("alter table trades rename to trades_bak") + engine.execute(f"alter table trades rename to {table_back_name}") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) @@ -112,8 +119,7 @@ def check_migrate(engine) -> None: stake_amount, amount, open_date, close_date, open_order_id, {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, {max_rate} max_rate - - from trades_bak + from {table_back_name} """) # Reread columns - the above recreated the table! diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 0cd2d089c..7ef4d2c25 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -7,6 +7,7 @@ from sqlalchemy import create_engine from freqtrade import constants, OperationalException from freqtrade.persistence import Trade, init, clean_dry_run_db +from freqtrade.tests.conftest import log_has @pytest.fixture(scope='function') @@ -405,7 +406,7 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.initial_stop_loss == 0.0 -def test_migrate_new(mocker, default_conf, fee): +def test_migrate_new(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ @@ -442,6 +443,9 @@ def test_migrate_new(mocker, default_conf, fee): # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) + + # fake previous backup + engine.execute("create table trades_bak as select * from trades") # Run init to test migration init(default_conf) @@ -456,6 +460,10 @@ def test_migrate_new(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "binance" + assert trade.max_rate == 0.0 + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert log_has("trying trades_bak1", caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): From 7cecae52799eb480e2265802e41ef0bd863d5a1c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 28 Jun 2018 14:23:07 +0200 Subject: [PATCH 193/239] Update ccxt from 1.14.288 to 1.14.289 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 62e2e3983..76dba3c49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.288 +ccxt==1.14.289 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 2d4ce593b511e3f9cd977a307c002307b4c6a052 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 19:48:05 +0200 Subject: [PATCH 194/239] catch crash with cobinhood fixes #959 --- freqtrade/freqtradebot.py | 11 +++++++---- freqtrade/tests/test_freqtradebot.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..985ecb370 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -277,11 +277,14 @@ class FreqtradeBot(object): return None min_stake_amounts = [] - if 'cost' in market['limits'] and 'min' in market['limits']['cost']: - min_stake_amounts.append(market['limits']['cost']['min']) + limits = market['limits'] + if ('cost' in limits and 'min' in limits['cost'] + and limits['cost']['min'] is not None): + min_stake_amounts.append(limits['cost']['min']) - if 'amount' in market['limits'] and 'min' in market['limits']['amount']: - min_stake_amounts.append(market['limits']['amount']['min'] * price) + if ('amount' in limits and 'min' in limits['amount'] + and limits['amount']['min'] is not None): + min_stake_amounts.append(limits['amount']['min'] * price) if not min_stake_amounts: return None diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..ff1b7161d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -348,6 +348,34 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) assert result is None + # no cost Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {"min": None}, + 'amount': {} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # no amount Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {}, + 'amount': {"min": None} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + # empty 'cost'/'amount' section mocker.patch( 'freqtrade.exchange.Exchange.get_markets', From 8ec9a0974978242fb1bd47587151eec6a0a5bda6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:22:43 +0200 Subject: [PATCH 195/239] Standardize retrier exception testing --- freqtrade/tests/exchange/test_exchange.py | 141 ++++++---------------- 1 file changed, 39 insertions(+), 102 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 53e59b34b..4d46d7e61 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,6 +14,22 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange +def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test exception handling """ + + with pytest.raises(TemporaryError): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) @@ -243,17 +259,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - with pytest.raises(TemporaryError): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == API_RETRY_COUNT + 1 - - with pytest.raises(OperationalException): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -282,15 +289,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() - - with pytest.raises(OperationalException): - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -345,15 +345,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) - - with pytest.raises(OperationalException): - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -416,17 +410,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('ABCD/BTC', default_conf['ticker_interval']) - - with pytest.raises(OperationalException): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) def test_get_ticker_history_sort(default_conf, mocker): @@ -515,24 +501,15 @@ def test_cancel_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == 123 - with pytest.raises(TemporaryError): - api_mock.cancel_order = MagicMock(side_effect=ccxt.NetworkError) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.cancel_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -550,23 +527,15 @@ def test_get_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.get_order('X', 'TKN/BTC') == 456 - with pytest.raises(TemporaryError): - api_mock.fetch_order = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.fetch_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -651,19 +620,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) def test_get_markets(default_conf, mocker, markets): @@ -677,19 +636,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - assert api_mock.fetch_markets.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -704,19 +652,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - assert api_mock.calculate_fee.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From ebbfc720b24c6a378dcf79b5b1c5205f8dd95dea Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:51:59 +0200 Subject: [PATCH 196/239] increase test coverage --- freqtrade/tests/exchange/test_exchange.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4d46d7e61..75394ca84 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,6 +232,12 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') + with pytest.raises(TemporaryError): + # api_mock.fetch_balance = MagicMock(return_value={}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) + exchange.get_balance(currency='BTC') + def test_get_balances_dry_run(default_conf, mocker): default_conf['dry_run'] = True @@ -624,6 +630,9 @@ def test_get_trades_for_order(default_conf, mocker): 'get_trades_for_order', 'fetch_my_trades', order_id=order_id, pair='LTC/BTC', since=since) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] + def test_get_markets(default_conf, mocker, markets): api_mock = MagicMock() From fe8a21681e3fd8af90093a9fd004e7acd2c51f77 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:56:37 +0200 Subject: [PATCH 197/239] add test for Not supported --- freqtrade/tests/exchange/test_exchange.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 75394ca84..012c55356 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,8 +232,7 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') - with pytest.raises(TemporaryError): - # api_mock.fetch_balance = MagicMock(return_value={}) + with pytest.raises(TemporaryError, match=r'.*balance due to malformed exchange response:.*'): exchange = get_patched_exchange(mocker, default_conf, api_mock) mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) exchange.get_balance(currency='BTC') @@ -420,6 +419,11 @@ def test_get_ticker_history(default_conf, mocker): "get_ticker_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_ticker_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + def test_get_ticker_history_sort(default_conf, mocker): api_mock = MagicMock() From 15c7854e7f79c03132e86d789f7725108aec4529 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:11:45 +0200 Subject: [PATCH 198/239] add test for exchange_has --- freqtrade/tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 012c55356..25c6fa0c2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -113,6 +113,20 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): Exchange(conf) +def test_exchangehas(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf) + assert not exchange.exchange_has('ASDFASDF') + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'deadbeef': True}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert exchange.exchange_has("deadbeef") + + type(api_mock).has = PropertyMock(return_value={'deadbeef': False}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert not exchange.exchange_has("deadbeef") + + def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) From dcdc18a33868a5630b5b747eb2773819154fb88c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:18:38 +0200 Subject: [PATCH 199/239] rename test-function --- freqtrade/tests/exchange/test_exchange.py | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 25c6fa0c2..712205893 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,8 +14,8 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): - """Function to test exception handling """ +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) @@ -278,8 +278,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_balances", "fetch_balance") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -308,8 +308,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_tickers", "fetch_tickers") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -364,9 +364,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -429,9 +429,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) @@ -531,9 +531,9 @@ def test_cancel_order(default_conf, mocker): exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "cancel_order", "cancel_order", - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -557,9 +557,9 @@ def test_get_order(default_conf, mocker): exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_order', 'fetch_order', - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -644,9 +644,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_trades_for_order', 'fetch_my_trades', - order_id=order_id, pair='LTC/BTC', since=since) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] @@ -663,8 +663,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_markets', 'fetch_markets') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -679,8 +679,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_fee', 'calculate_fee') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From cf6b1a637ac0a2312c15d926093851a3a1adf4f1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:32:28 +0200 Subject: [PATCH 200/239] increase exchange code coverage --- freqtrade/tests/exchange/test_exchange.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 712205893..99b960885 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -36,7 +36,7 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) -def test_init_exception(default_conf): +def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' with pytest.raises( @@ -44,6 +44,13 @@ def test_init_exception(default_conf): match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): Exchange(default_conf) + default_conf['exchange']['name'] = 'binance' + with pytest.raises( + OperationalException, + match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) + Exchange(default_conf) + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() From 8a941f3aa81ee6aad4778be0e1ae0952ce3108c9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 29 Jun 2018 14:23:06 +0200 Subject: [PATCH 201/239] Update ccxt from 1.14.289 to 1.14.295 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 76dba3c49..6995c2513 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.289 +ccxt==1.14.295 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 0ce08932ed31edde076b91589526ffcd06f428be Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 30 Jun 2018 09:54:31 +0300 Subject: [PATCH 202/239] mypy fixes --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b8b53ab3c..eb9015356 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -62,7 +62,7 @@ class Hyperopt(Backtesting): # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = [] + self.trials: List = [] def get_args(self, params): dimensions = self.hyperopt_space() @@ -191,7 +191,7 @@ class Hyperopt(Backtesting): ] @staticmethod - def indicator_space() -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters """ @@ -349,7 +349,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - self.exchange = None + self.exchange = None # type: ignore self.load_previous_results() cpus = multiprocessing.cpu_count() From 98108a78f1ff5ccd0ca4390a20885137f2d65637 Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 09:32:52 +0800 Subject: [PATCH 203/239] separating unfulfilled timeouts for buy and sell --- config.json.example | 5 ++++- config_full.json.example | 5 ++++- freqtrade/constants.py | 9 ++++++++- freqtrade/freqtradebot.py | 19 ++++++++++++------- freqtrade/tests/conftest.py | 5 ++++- freqtrade/tests/test_freqtradebot.py | 8 ++++---- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/config.json.example b/config.json.example index e5bdc95b1..336a0c374 100644 --- a/config.json.example +++ b/config.json.example @@ -5,7 +5,10 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 231384acc..1985edc70 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -12,7 +12,10 @@ "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..44bd64fe9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -61,7 +61,14 @@ CONF_SCHEMA = { 'minProperties': 1 }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'unfilledtimeout': { + 'type': 'object', + 'properties': { + 'use_book_order': {'type': 'boolean'}, + 'buy': {'type': 'number', 'minimum': 3}, + 'sell': {'type': 'number', 'minimum': 10} + } + }, 'bid_strategy': { 'type': 'object', 'properties': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..c5f932119 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -160,7 +160,7 @@ class FreqtradeBot(object): if 'unfilledtimeout' in self.config: # Check and handle any timed out open orders - self.check_handle_timedout(self.config['unfilledtimeout']) + self.check_handle_timedout() Trade.session.flush() except TemporaryError as error: @@ -492,13 +492,16 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False - def check_handle_timedout(self, timeoutvalue: int) -> None: + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary :param timeoutvalue: Number of minutes until order is considered timed out :return: None """ - timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime + buy_timeout = self.config['unfilledtimeout']['buy'] + sell_timeout = self.config['unfilledtimeout']['sell'] + buy_timeoutthreashold = arrow.utcnow().shift(minutes=-buy_timeout).datetime + sell_timeoutthreashold = arrow.utcnow().shift(minutes=-sell_timeout).datetime for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): try: @@ -521,10 +524,12 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ if int(order['remaining']) == 0: continue - if order['side'] == 'buy' and ordertime < timeoutthreashold: - self.handle_timedout_limit_buy(trade, order) - elif order['side'] == 'sell' and ordertime < timeoutthreashold: - self.handle_timedout_limit_sell(trade, order) + # Check if trade is still actually open + if (order['status'] == 'open'): + if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: + self.handle_timedout_limit_buy(trade, order) + elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: + self.handle_timedout_limit_sell(trade, order) # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d9ccaa325..4877fe556 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -100,7 +100,10 @@ def default_conf(): "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..fb7579d45 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1124,7 +1124,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe Trade.session.add(trade_buy) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1165,7 +1165,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, Trade.session.add(trade_sell) # check it does cancel sell orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert trade_sell.is_open is True @@ -1205,7 +1205,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # check it does cancel buy orders over the time limit # note this is for a partially-complete buy order - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1256,7 +1256,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - 'recent call last):\n.*' ) - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert filter(regexp.match, caplog.record_tuples) From c447644fd16386868c6a4a1aa6b8a5e139af8c22 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:06 +0200 Subject: [PATCH 204/239] Update ccxt from 1.14.295 to 1.14.301 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6995c2513..72decf95a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.295 +ccxt==1.14.301 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 5a591e01c08fa0c60db152da2c04ec04dfd4e58a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:07 +0200 Subject: [PATCH 205/239] Update sqlalchemy from 1.2.8 to 1.2.9 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72decf95a..50441879a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.14.301 -SQLAlchemy==1.2.8 +SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 From 14e12bd3c09c42a4ed20c170765a55826e304b04 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:37:34 +0200 Subject: [PATCH 206/239] Fix missing comma in example.json --- config.json.example | 2 +- config_full.json.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 336a0c374..606167ea3 100644 --- a/config.json.example +++ b/config.json.example @@ -8,7 +8,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 1985edc70..4bb794937 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -15,7 +15,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, From 9e3e900f78d5105f6d145d80fd5d97dbea0b9cac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:49:46 +0200 Subject: [PATCH 207/239] Add get_markets mock to new tests --- freqtrade/tests/test_freqtradebot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..f66864946 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1599,6 +1599,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -1616,7 +1617,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert freqtrade.handle_trade(trade) is True -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1634,6 +1635,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) From 8f49d5eb1002230fb424e4b505ec5f9561e778ea Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 18:49:43 +0800 Subject: [PATCH 208/239] documentation updates --- docs/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5279ed06e..ea3e99f15 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,7 +22,8 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. +| `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. +| `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. From c66f858b988bd18ee24045839211f4248640b097 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:37:55 +0200 Subject: [PATCH 209/239] rename innerfun to mock_ccxt_fun --- freqtrade/tests/exchange/test_exchange.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 99b960885..6cf7f11b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,20 +14,20 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 def test_init(default_conf, mocker, caplog): From 2dc881558d65b19dd6596d3a8847da618f7d6885 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:41:19 +0200 Subject: [PATCH 210/239] address PR comments --- config.json.example | 4 ++-- config_full.json.example | 4 ++-- freqtrade/constants.py | 1 - freqtrade/freqtradebot.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index 606167ea3..79ba2ac64 100644 --- a/config.json.example +++ b/config.json.example @@ -6,8 +6,8 @@ "ticker_interval" : "5m", "dry_run": false, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/config_full.json.example b/config_full.json.example index 4bb794937..d40c11a91 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -13,8 +13,8 @@ }, "stoploss": -0.10, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 44bd64fe9..04d4a44ac 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -64,7 +64,6 @@ CONF_SCHEMA = { 'unfilledtimeout': { 'type': 'object', 'properties': { - 'use_book_order': {'type': 'boolean'}, 'buy': {'type': 'number', 'minimum': 3}, 'sell': {'type': 'number', 'minimum': 10} } diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c5f932119..4d6352e55 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -525,7 +525,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ continue # Check if trade is still actually open - if (order['status'] == 'open'): + if order['status'] == 'open': if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: self.handle_timedout_limit_buy(trade, order) elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: From e39d88ef65e72387f18198da269a2d7660a2d6c6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:54:26 +0200 Subject: [PATCH 211/239] Address some PR comments --- freqtrade/constants.py | 2 +- freqtrade/persistence.py | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4ba1a7094..d80eea6f4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -62,7 +62,7 @@ CONF_SCHEMA = { }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, 'trailing_stop': {'type': 'boolean'}, - 'trailing_stop_positive': {'type': 'number', 'minimum': 0}, + 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, 'bid_strategy': { 'type': 'object', diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 2b226e53a..ad90d3f92 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -184,14 +184,8 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) - def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool =False): - """ - this adjusts the stop loss to it's most recently observed setting - :param current_price: - :param stoploss in percent: - :param initial: - :return: - """ + def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): + """this adjusts the stop loss to it's most recently observed setting""" if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do @@ -207,7 +201,7 @@ class Trade(_DECL_BASE): self.max_rate = current_price # no stop loss assigned yet - if self.stop_loss is None or self.stop_loss == 0: + if not self.stop_loss: logger.debug("assigning new stop loss") self.stop_loss = new_loss self.initial_stop_loss = new_loss From 937644a04b96adbdb26ed54c5969945e09f314c2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:55:51 +0200 Subject: [PATCH 212/239] change while-loop to enumerate - add intensified test for this scenario --- freqtrade/tests/test_persistence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 7ef4d2c25..12ae6579f 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -446,6 +446,8 @@ def test_migrate_new(mocker, default_conf, fee, caplog): # fake previous backup engine.execute("create table trades_bak as select * from trades") + + engine.execute("create table trades_bak1 as select * from trades") # Run init to test migration init(default_conf) @@ -464,6 +466,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 assert log_has("trying trades_bak1", caplog.record_tuples) + assert log_has("trying trades_bak2", caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): From 782570e71eb6fca9c5a70cf36896d692661a14a7 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 20:03:07 +0200 Subject: [PATCH 213/239] Address PR comment --- config.json.example | 2 +- freqtrade/persistence.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 87c27090e..e8473e919 100644 --- a/config.json.example +++ b/config.json.example @@ -36,7 +36,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "sell_fullfilled_at_roi": false + "ignore_roi_if_buy_signal": false }, "telegram": { "enabled": true, diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ad90d3f92..dfeb6145c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -80,7 +80,7 @@ def check_migrate(engine) -> None: tabs = inspector.get_table_names() table_back_name = 'trades_bak' i = 0 - while table_back_name in tabs: + for i, table_back_name in enumerate(tabs): i += 1 table_back_name = f'trades_bak{i}' logger.info(f'trying {table_back_name}') From 3c5be55eb907878c73f0cfb26a4f21b3af2a4745 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 20:17:30 +0200 Subject: [PATCH 214/239] remove unnecessary variable --- freqtrade/persistence.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index dfeb6145c..aa33f7063 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -79,9 +79,7 @@ def check_migrate(engine) -> None: cols = inspector.get_columns('trades') tabs = inspector.get_table_names() table_back_name = 'trades_bak' - i = 0 for i, table_back_name in enumerate(tabs): - i += 1 table_back_name = f'trades_bak{i}' logger.info(f'trying {table_back_name}') From a58d51ded0fcfd06cbe7344da202e353e9deff10 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 2 Jul 2018 09:56:58 +0300 Subject: [PATCH 215/239] update hyperopt documentation --- docs/hyperopt.md | 279 ++++++++++++++++------------------------------- 1 file changed, 96 insertions(+), 183 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 2ad94896a..8d3fe6704 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,153 +1,109 @@ # Hyperopt This page explains how to tune your strategy by finding the optimal -parameters with Hyperopt. +parameters, process called hyperparameter optimization. The bot uses several +algorithms included in `scikit-optimize` package to accomplish this. The +search will burn all your CPU cores, make your laptop sound like a fighter jet +and still take a long time. ## Table of Contents - [Prepare your Hyperopt](#prepare-hyperopt) - - [1. Configure your Guards and Triggers](#1-configure-your-guards-and-triggers) - - [2. Update the hyperopt config file](#2-update-the-hyperopt-config-file) -- [Advanced Hyperopt notions](#advanced-notions) - - [Understand the Guards and Triggers](#understand-the-guards-and-triggers) +- [Configure your Guards and Triggers](#configure-your-guards-and-triggers) +- [Solving a Mystery](#solving-a-mystery) +- [Adding New Indicators](#adding-new-indicators) - [Execute Hyperopt](#execute-hyperopt) - [Understand the hyperopts result](#understand-the-backtesting-result) -## Prepare Hyperopt -Before we start digging in Hyperopt, we recommend you to take a look at -your strategy file located into [user_data/strategies/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +## Prepare Hyperopting +We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) -### 1. Configure your Guards and Triggers +### Configure your Guards and Triggers There are two places you need to change in your strategy file to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L278-L294). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297) known as `SPACE`. +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L278-L294). +- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L218-L229) +and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. -There you have two different type of indicators: 1. `guards` and 2. -`triggers`. +There you have two different type of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or buy when close price touches lower bollinger band. -HyperOpt will, for each eval round, pick just ONE trigger, and possibly -multiple guards. So that the constructed strategy will be something like +Hyperoptimization will, for each eval round, pick one trigger and possibly +multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower bollinger band, BUT only if ADX > 10*". - -If you have updated the buy strategy, means change the content of +If you have updated the buy strategy, ie. changed the contents of `populate_buy_trend()` method you have to update the `guards` and -`triggers` hyperopts must used. +`triggers` hyperopts must use. -As for an example if your `populate_buy_trend()` method is: -```python -def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - dataframe.loc[ - (dataframe['rsi'] < 35) & - (dataframe['adx'] > 65), - 'buy'] = 1 +## Solving a Mystery - return dataframe -``` +Let's say you are curious: should you use MACD crossings or lower Bollinger +Bands to trigger your buys. And you also wonder should you use RSI or ADX to +help with those buy decisions. If you decide to use RSI or ADX, which values +should I use for them? So let's use hyperparameter optimization to solve this +mystery. -Your hyperopt file must contain `guards` to find the right value for -`(dataframe['adx'] > 65)` & and `(dataframe['plus_di'] > 0.5)`. That -means you will need to enable/disable triggers. - -In our case the `SPACE` and `populate_buy_trend` in your strategy file -will look like: -```python -space = { - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema5_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'stochf_cross'}, - {'type': 'ht_sine'}, - ]), -} - -... - -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - - # TRIGGERS - triggers = { - 'lower_bb': dataframe['tema'] <= dataframe['blower'], - 'faststoch10': (crossed_above(dataframe['fastd'], 10.0)), - 'ao_cross_zero': (crossed_above(dataframe['ao'], 0.0)), - 'ema5_cross_ema10': (crossed_above(dataframe['ema5'], dataframe['ema10'])), - 'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])), - 'sar_reversal': (crossed_above(dataframe['close'], dataframe['sar'])), - 'stochf_cross': (crossed_above(dataframe['fastk'], dataframe['fastd'])), - 'ht_sine': (crossed_above(dataframe['htleadsine'], dataframe['htsine'])), - } - ... -``` - - -### 2. Update the hyperopt config file -Hyperopt is using a dedicated config file. Currently hyperopt -cannot use your config file. It is also made on purpose to allow you -testing your strategy with different configurations. - -The Hyperopt configuration is located in -[user_data/hyperopt_conf.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopt_conf.py). - - -## Advanced notions -### Understand the Guards and Triggers -When you need to add the new guards and triggers to be hyperopt -parameters, you do this by adding them into the [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297). - -If it's a trigger, you add one line to the 'trigger' choice group and that's it. - -If it's a guard, you will add a line like this: -``` -'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} -]), -``` -This says, "*one of the guards is RSI, it can have two values, enabled or -disabled. If it is enabled, try different values for it between 20 and 40*". - -So, the part of the strategy builder using the above setting looks like -this: +We will start by defining a search space: ``` -if params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(20, 40, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') + ] ``` -It checks if Hyperopt wants the RSI guard to be enabled for this -round `params['rsi']['enabled']` and if it is, then it will add a -condition that says RSI must be smaller than the value hyperopt picked -for this evaluation, which is given in the `params['rsi']['value']`. +Above definition says: I have five parameters I want you to randomly combine +to find the best combination. Two of them are integer values (`adx-value` +and `rsi-value`) and I want you test in the range of values 20 to 40. +Then we have three category variables. First two are either `True` or `False`. +We use these to either enable or disable the ADX and RSI guards. The last +one we call `trigger` and use it to decide which buy trigger we want to use. -That's it. Now you can add new parts of strategies to Hyperopt and it -will try all the combinations with all different values in the search -for best working algo. +So let's write the buy strategy using these values: +``` + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) -### Add a new Indicators + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend +``` + +Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) +with different value combinations. It will then use the given historical data and make +buys based on the buy signals generated with the above function and based on the results +it will end with telling you which paramter combination produced the best profits. + +### Adding New Indicators If you want to test an indicator that isn't used by the bot currently, you need to add it to the `populate_indicators()` method in `hyperopt.py`. @@ -164,12 +120,12 @@ python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. -### Execute hyperopt with different ticker-data source +### Execute Hyperopt with Different Ticker-Data Source If you would like to hyperopt parameters using an alternate ticker data that you have on-disk, use the `--datadir PATH` option. Default hyperopt will use data from directory `user_data/data`. -### Running hyperopt with smaller testset +### Running Hyperopt with Smaller Testset Use the `--timeperiod` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: @@ -178,7 +134,7 @@ Example: python3 ./freqtrade/main.py hyperopt --timeperiod -200 ``` -### Running hyperopt with smaller search space +### Running Hyperopt with Smaller Search Space Use the `--spaces` argument to limit the search space used by hyperopt. Letting Hyperopt optimize everything is a huuuuge search space. Often it might make more sense to start by just searching for initial buy algorithm. @@ -193,87 +149,44 @@ Legal values are: - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` -## Understand the hyperopts result -Once Hyperopt is completed you can use the result to adding new buy -signal. Given following result from hyperopt: -``` -Best parameters: -{ - "adx": { - "enabled": true, - "value": 15.0 - }, - "fastd": { - "enabled": true, - "value": 40.0 - }, - "green_candle": { - "enabled": true - }, - "mfi": { - "enabled": false - }, - "over_sar": { - "enabled": false - }, - "rsi": { - "enabled": true, - "value": 37.0 - }, - "trigger": { - "type": "lower_bb" - }, - "uptrend_long_ema": { - "enabled": true - }, - "uptrend_short_ema": { - "enabled": false - }, - "uptrend_sma": { - "enabled": false - } -} +## Understand the Hyperopts Result +Once Hyperopt is completed you can use the result to creating a new strategy. +Given following result from hyperopt: -Best Result: - 2197 trades. Avg profit 1.84%. Total profit 0.79367541 BTC. Avg duration 241.0 mins. +``` +Best result: + 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. +with values: +{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower'} ``` You should understand this result like: -- You should **consider** the guard "adx" (`"adx"` is `"enabled": true`) -and the best value is `15.0` (`"value": 15.0,`) -- You should **consider** the guard "fastd" (`"fastd"` is `"enabled": -true`) and the best value is `40.0` (`"value": 40.0,`) -- You should **consider** to enable the guard "green_candle" -(`"green_candle"` is `"enabled": true`) but this guards as no -customizable value. -- You should **ignore** the guard "mfi" (`"mfi"` is `"enabled": false`) -- and so on... +- The buy trigger that worked best was `bb_lower`. +- You should not use ADX because `adx-enabled: False`) +- You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) You have to look inside your strategy file into `buy_strategy_generator()` method, what those values match to. -So for example you had `adx:` with the `value: 15.0` so we would look -at `adx`-block, that translates to the following code block: +So for example you had `rsi-value: 29.0` so we would look +at `rsi`-block, that translates to the following code block: ``` -(dataframe['adx'] > 15.0) +(dataframe['rsi'] < 29.0) ``` -Translating your whole hyperopt result to as the new buy-signal -would be the following: +Translating your whole hyperopt result as the new buy-signal +would then look like: ``` def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: dataframe.loc[ ( - (dataframe['adx'] > 15.0) & # adx-value - (dataframe['fastd'] < 40.0) & # fastd-value - (dataframe['close'] > dataframe['open']) & # green_candle - (dataframe['rsi'] < 37.0) & # rsi-value - (dataframe['ema50'] > dataframe['ema100']) # uptrend_long_ema + (dataframe['rsi'] < 29.0) & # rsi-value + dataframe['close'] < dataframe['bb_lowerband'] # trigger ), 'buy'] = 1 return dataframe ``` -## Next step +## Next Step Now you have a perfect bot and want to control it from Telegram. Your next step is to learn the [Telegram usage](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md). From fa8fc3e4ce19b566a9ea5f079cb26db38b143089 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 2 Jul 2018 11:44:33 +0300 Subject: [PATCH 216/239] handle the case where we have zero buys --- freqtrade/optimize/hyperopt.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index eb9015356..e499def70 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -31,6 +31,8 @@ from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) +MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization + class Hyperopt(Backtesting): """ @@ -152,7 +154,8 @@ class Hyperopt(Backtesting): trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) profit_loss = max(0, 1 - total_profit / self.expected_max_profit) duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) - return trade_loss + profit_loss + duration_loss + result = trade_loss + profit_loss + duration_loss + return result @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: @@ -293,6 +296,13 @@ class Hyperopt(Backtesting): trade_count = len(results.index) trade_duration = results.trade_duration.mean() + if trade_count == 0: + return { + 'loss': MAX_LOSS, + 'params': params, + 'result': result_explanation, + } + loss = self.calculate_loss(total_profit, trade_count, trade_duration) return { From 76343ecb77e2869a80f76adabf748baeeb314e1a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 2 Jul 2018 14:23:06 +0200 Subject: [PATCH 217/239] Update ccxt from 1.14.301 to 1.15.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50441879a..ebb5b5b49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.301 +ccxt==1.15.3 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 79aab4cce2f516df4d9504d99c31f3ccf4b70394 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 11:17:41 +0300 Subject: [PATCH 218/239] use fstring --- freqtrade/optimize/hyperopt.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e499def70..9b26b5b09 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -71,10 +71,8 @@ class Hyperopt(Backtesting): # Ensure the number of dimensions match # the number of parameters in the list x. if len(params) != len(dimensions): - msg = "Mismatch in number of search-space dimensions. " \ - "len(dimensions)=={} and len(x)=={}" - msg = msg.format(len(dimensions), len(params)) - raise ValueError(msg) + raise ValueError('Mismatch in number of search-space dimensions. ' + f'len(dimensions)=={len(dimensions)} and len(x)=={len(params)}') # Create a dict where the keys are the names of the dimensions # and the values are taken from the list of parameters x. From 2713fdb8607b01602d34c37e7c31facc7f9cb78e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 11:46:56 +0300 Subject: [PATCH 219/239] use cpu count explicitly in job count --- freqtrade/optimize/hyperopt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 9b26b5b09..8290eb2d8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -323,13 +323,13 @@ class Hyperopt(Backtesting): results.trade_duration.mean(), ) - def get_optimizer(self) -> Optimizer: + def get_optimizer(self, cpu_count) -> Optimizer: return Optimizer( self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, - acq_optimizer_kwargs={'n_jobs': -1} + acq_optimizer_kwargs={'n_jobs': cpu_count} ) def run_optimizer_parallel(self, parallel, asked) -> List: @@ -361,11 +361,11 @@ class Hyperopt(Backtesting): self.load_previous_results() cpus = multiprocessing.cpu_count() - logger.info(f'Found {cpus}. Let\'s make them scream!') + logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') - opt = self.get_optimizer() + opt = self.get_optimizer(cpus) try: - with Parallel(n_jobs=-1) as parallel: + with Parallel(n_jobs=cpus) as parallel: for i in range(self.total_tries//cpus): asked = opt.ask(n_points=cpus) f_val = self.run_optimizer_parallel(parallel, asked) From 4a26b88a17af950f2bdedcfde43e2cd684dfcd4d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 12:43:25 +0300 Subject: [PATCH 220/239] improve documentation --- docs/hyperopt.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8d3fe6704..f4b69b632 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,7 +1,7 @@ # Hyperopt -This page explains how to tune your strategy by finding the optimal -parameters, process called hyperparameter optimization. The bot uses several -algorithms included in `scikit-optimize` package to accomplish this. The +This page explains how to tune your strategy by finding the optimal +parameters, a process called hyperparameter optimization. The bot uses several +algorithms included in the `scikit-optimize` package to accomplish this. The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. @@ -17,18 +17,17 @@ and still take a long time. We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) ### Configure your Guards and Triggers -There are two places you need to change in your strategy file to add a -new buy strategy for testing: +There are two places you need to change to add a new buy strategy for testing: - Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L278-L294). - Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L218-L229) and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. There you have two different type of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or never buy if -current price is over EMA10. +1. Guards are conditions like "never buy if ADX < 10", or "never buy if +current price is over EMA10". 2. Triggers are ones that actually trigger buy in specific moment, like -"buy when EMA5 crosses over EMA10" or buy when close price touches lower -bollinger band. +"buy when EMA5 crosses over EMA10" or "buy when close price touches lower +bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like @@ -103,9 +102,13 @@ with different value combinations. It will then use the given historical data an buys based on the buy signals generated with the above function and based on the results it will end with telling you which paramter combination produced the best profits. -### Adding New Indicators -If you want to test an indicator that isn't used by the bot currently, -you need to add it to the `populate_indicators()` method in `hyperopt.py`. +The search for best parameters starts with a few random combinations and then uses a +regressor algorithm (currently ExtraTreesRegressor) to quickly find a parameter combination +that minimizes the value of the objective function `calculate_loss` in `hyperopt.py`. + +The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. +When you want to test an indicator that isn't used by the bot currently, remember to +add it to the `populate_indicators()` method in `hyperopt.py`. ## Execute Hyperopt Once you have updated your hyperopt configuration you can run it. @@ -150,8 +153,8 @@ Legal values are: - space-separated list of any of the above values for example `--spaces roi stoploss` ## Understand the Hyperopts Result -Once Hyperopt is completed you can use the result to creating a new strategy. -Given following result from hyperopt: +Once Hyperopt is completed you can use the result to create a new strategy. +Given the following result from hyperopt: ``` Best result: From ee4754cfb985fbe608ff02b21c65102223e1e005 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 14:46:16 +0300 Subject: [PATCH 221/239] avoid re-serialization of whole dataframe --- freqtrade/optimize/hyperopt.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8290eb2d8..57bf75742 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,14 +14,14 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, Optional, List +from typing import Dict, Any, Callable, List import talib.abstract as ta from pandas import DataFrame from skopt.space import Real, Integer, Categorical, Dimension from skopt import Optimizer -from sklearn.externals.joblib import Parallel, delayed +from sklearn.externals.joblib import Parallel, delayed, load, dump import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments @@ -32,6 +32,7 @@ from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization +TICKERDATA_PICKLE = os.path.join('user_data', 'hyperopt_tickerdata.pkl') class Hyperopt(Backtesting): @@ -60,7 +61,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Configuration and data used by hyperopt - self.processed: Optional[Dict[str, Any]] = None +# self.processed: Optional[Dict[str, Any]] = None # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') @@ -281,10 +282,11 @@ class Hyperopt(Backtesting): if self.has_space('stoploss'): self.analyze.strategy.stoploss = params['stoploss'] + processed = load(TICKERDATA_PICKLE) results = self.backtest( { 'stake_amount': self.config['stake_amount'], - 'processed': self.processed, + 'processed': processed, 'realistic': self.config.get('realistic_simulation', False), } ) @@ -356,7 +358,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore - self.processed = self.tickerdata_to_dataframe(data) + dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() From 2c0e950486ef8e7368160b35934ca0c817f3d0a9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 3 Jul 2018 14:23:05 +0200 Subject: [PATCH 222/239] Update ccxt from 1.15.3 to 1.15.7 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ebb5b5b49..98721b74c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.3 +ccxt==1.15.7 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From ef59f9ad249f8a24a0ab29b2cb4c2f28d180c206 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:50:24 +0300 Subject: [PATCH 223/239] sort imports in hyperopt.py --- freqtrade/optimize/hyperopt.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 57bf75742..a1d86bd18 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,23 +5,21 @@ This module contains the hyperopt logic """ import logging +import multiprocessing import os import pickle import sys -import multiprocessing - from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, List +from typing import Any, Callable, Dict, List import talib.abstract as ta from pandas import DataFrame - -from skopt.space import Real, Integer, Categorical, Dimension +from sklearn.externals.joblib import Parallel, delayed, dump, load from skopt import Optimizer -from sklearn.externals.joblib import Parallel, delayed, load, dump +from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments From 2cde540645bf321360594769768f017c2ceabe0f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:50:45 +0300 Subject: [PATCH 224/239] remove dead code --- freqtrade/optimize/hyperopt.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a1d86bd18..861523965 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -58,9 +58,6 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 - # Configuration and data used by hyperopt -# self.processed: Optional[Dict[str, Any]] = None - # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') self.trials: List = [] From 3a7056ea1bab0a2cea913e2eca3c6e590481ee93 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:54:32 +0300 Subject: [PATCH 225/239] run at least one epoch --- freqtrade/optimize/hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 861523965..71e923f00 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -361,9 +361,10 @@ class Hyperopt(Backtesting): logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') opt = self.get_optimizer(cpus) + EVALS = max(self.total_tries//cpus, 1) try: with Parallel(n_jobs=cpus) as parallel: - for i in range(self.total_tries//cpus): + for i in range(EVALS): asked = opt.ask(n_points=cpus) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) From 9dbe0f50a3e6a708a621bea70757bf9e0b6f7577 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:09:59 +0300 Subject: [PATCH 226/239] fix tests after changing the dumping and pickling dataframe in hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9194a66c9..bbe16ae7a 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -187,6 +187,7 @@ def test_roi_table_generation(init_hyperopt) -> None: def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) parallel = mocker.patch( @@ -208,6 +209,7 @@ def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> N parallel.assert_called_once() assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text + assert dumper.called def test_format_results(init_hyperopt): @@ -316,6 +318,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: MagicMock(return_value=backtest_result) ) patch_exchange(mocker) + mocker.patch('freqtrade.optimize.hyperopt.load', MagicMock()) optimizer_param = { 'adx-value': 0, From c4a8435e007db87e157cf65e43213c2b4da81129 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:17:43 +0300 Subject: [PATCH 227/239] change pickle file name to better suit it's current purpose --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 71e923f00..d90468cff 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -59,7 +59,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Previous evaluations - self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials_file = os.path.join('user_data', 'hyperopt_results.pickle') self.trials: List = [] def get_args(self, params): From 96bb2efe69ded2e6cbd2998804445e9d8726ded0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:51:48 +0300 Subject: [PATCH 228/239] use joblib.dump and load for trials --- freqtrade/optimize/hyperopt.py | 5 ++--- freqtrade/tests/optimize/test_hyperopt.py | 9 +++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d90468cff..7138e2601 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,7 +7,6 @@ This module contains the hyperopt logic import logging import multiprocessing import os -import pickle import sys from argparse import Namespace from functools import reduce @@ -99,14 +98,14 @@ class Hyperopt(Backtesting): """ if self.trials: logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) - pickle.dump(self.trials, open(self.trials_file, 'wb')) + dump(self.trials, self.trials_file) def read_trials(self) -> List: """ Read hyperopt trials file """ logger.info('Reading Trials from \'%s\'', self.trials_file) - trials = pickle.load(open(self.trials_file, 'rb')) + trials = load(self.trials_file) os.remove(self.trials_file) return trials diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index bbe16ae7a..72a102c22 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -39,7 +39,7 @@ def create_trials(mocker) -> None: mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1) mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) return [{'loss': 1, 'result': 'foo', 'params': {}}] @@ -139,10 +139,9 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) - mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) hyperopt = _HYPEROPT - mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) _HYPEROPT.trials = trials hyperopt.save_trials() @@ -157,8 +156,7 @@ def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials) - mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) hyperopt = _HYPEROPT hyperopt_trial = hyperopt.read_trials() @@ -168,7 +166,6 @@ def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: caplog.record_tuples ) assert hyperopt_trial == trials - mock_open.assert_called_once() mock_load.assert_called_once() From bf4d0a9b7088eb012c1090144d52569c791848b6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 4 Jul 2018 10:31:35 +0300 Subject: [PATCH 229/239] sort imports --- freqtrade/__main__.py | 2 +- freqtrade/analyze.py | 3 +-- freqtrade/arguments.py | 5 +++-- freqtrade/configuration.py | 8 ++++---- freqtrade/fiat_convert.py | 4 +++- freqtrade/freqtradebot.py | 8 +++----- freqtrade/indicator_helpers.py | 2 +- freqtrade/misc.py | 2 +- freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/persistence.py | 5 ++--- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/strategy/interface.py | 2 +- freqtrade/strategy/resolver.py | 3 +-- freqtrade/tests/conftest.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 8 ++++---- freqtrade/tests/optimize/test_backtesting.py | 7 ++++--- freqtrade/tests/optimize/test_optimize.py | 15 +++++++++------ freqtrade/tests/rpc/test_rpc.py | 3 ++- freqtrade/tests/rpc/test_rpc_manager.py | 2 +- freqtrade/tests/rpc/test_rpc_telegram.py | 11 ++++++----- freqtrade/tests/test_acl_pair.py | 3 ++- freqtrade/tests/test_analyze.py | 4 ++-- freqtrade/tests/test_configuration.py | 6 +++--- freqtrade/tests/test_fiat_convert.py | 1 - freqtrade/tests/test_freqtradebot.py | 6 ++++-- freqtrade/tests/test_indicator_helpers.py | 2 +- freqtrade/tests/test_main.py | 2 +- freqtrade/tests/test_misc.py | 4 ++-- freqtrade/tests/test_persistence.py | 4 ++-- 29 files changed, 71 insertions(+), 65 deletions(-) diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py index fe1318a35..7d271dfd1 100644 --- a/freqtrade/__main__.py +++ b/freqtrade/__main__.py @@ -7,8 +7,8 @@ To launch Freqtrade as a module """ import sys -from freqtrade import main +from freqtrade import main if __name__ == '__main__': main.set_loggers() diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index bf6b89b17..6d6a85596 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -12,8 +12,7 @@ from pandas import DataFrame, to_datetime from freqtrade import constants from freqtrade.exchange import Exchange from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver, IStrategy - +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 8ce7c546d..731c5d88c 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -2,12 +2,13 @@ This module contains the argument manager class """ -import os import argparse import logging +import os import re +from typing import List, NamedTuple, Optional + import arrow -from typing import List, Optional, NamedTuple from freqtrade import __version__, constants diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 7c3a5eb4b..c90eddeb8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -1,18 +1,18 @@ """ This module contains the configuration class """ -import os import json import logging +import os from argparse import Namespace -from typing import Optional, Dict, Any +from typing import Any, Dict, Optional + +import ccxt from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match -import ccxt from freqtrade import OperationalException, constants - logger = logging.getLogger(__name__) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 44a4f3054..2e1a7cac8 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -7,10 +7,12 @@ import logging import time from typing import Dict, List -from coinmarketcap import Market from requests.exceptions import RequestException +from coinmarketcap import Market + from freqtrade.constants import SUPPORTED_FIAT + logger = logging.getLogger(__name__) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 530535710..64ff170fd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,16 +7,14 @@ import logging import time import traceback from datetime import datetime -from typing import Dict, List, Optional, Any, Callable +from typing import Any, Callable, Dict, List, Optional import arrow import requests from cachetools import TTLCache, cached -from freqtrade import ( - DependencyException, OperationalException, TemporaryError, persistence, __version__, -) -from freqtrade import constants +from freqtrade import (DependencyException, OperationalException, + TemporaryError, __version__, constants, persistence) from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter diff --git a/freqtrade/indicator_helpers.py b/freqtrade/indicator_helpers.py index 50586578a..f8ea0d939 100644 --- a/freqtrade/indicator_helpers.py +++ b/freqtrade/indicator_helpers.py @@ -1,4 +1,4 @@ -from math import exp, pi, sqrt, cos +from math import cos, exp, pi, sqrt import numpy as np import talib as ta diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 90a1db42b..832437951 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,10 +2,10 @@ Various tool function for Freqtrade and scripts """ +import gzip import json import logging import re -import gzip from datetime import datetime from typing import Dict diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 316fba508..16c21258f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -7,18 +7,18 @@ import logging import operator from argparse import Namespace from datetime import datetime -from typing import Dict, Tuple, Any, List, Optional, NamedTuple +from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize -from freqtrade import constants, DependencyException -from freqtrade.exchange import Exchange +from freqtrade import DependencyException, constants from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aa33f7063..764ba6509 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,12 +5,11 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional, Any +from typing import Any, Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine) -from sqlalchemy import inspect + create_engine, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ee6ecb770..f8cb136e4 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -3,9 +3,9 @@ This module contains class to define a RPC communications """ import logging from abc import abstractmethod -from datetime import datetime, timedelta, date +from datetime import date, datetime, timedelta from decimal import Decimal -from typing import Dict, Tuple, Any, List +from typing import Any, Dict, List, Tuple import arrow import sqlalchemy as sql diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4ae358c6f..f73617f46 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -2,8 +2,8 @@ IStrategy interface This module defines the interface to apply for strategies """ -from typing import Dict from abc import ABC, abstractmethod +from typing import Dict from pandas import DataFrame diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 0dcd3fc6a..10cedb073 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -8,13 +8,12 @@ import inspect import logging import os from collections import OrderedDict -from typing import Optional, Dict, Type +from typing import Dict, Optional, Type from freqtrade import constants from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy - logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 4877fe556..9c86d1ece 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,8 +2,8 @@ import json import logging from datetime import datetime -from typing import Dict, Optional from functools import reduce +from typing import Dict, Optional from unittest.mock import MagicMock import arrow @@ -11,8 +11,8 @@ import pytest from jsonschema import validate from telegram import Chat, Message, Update -from freqtrade.analyze import Analyze from freqtrade import constants +from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6cf7f11b0..3ddec0ded 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -2,16 +2,16 @@ # pragma pylint: disable=protected-access import logging from copy import deepcopy -from random import randint from datetime import datetime +from random import randint from unittest.mock import MagicMock, PropertyMock import ccxt import pytest -from freqtrade import OperationalException, DependencyException, TemporaryError -from freqtrade.exchange import Exchange, API_RETRY_COUNT -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade import DependencyException, OperationalException, TemporaryError +from freqtrade.exchange import API_RETRY_COUNT, Exchange +from freqtrade.tests.conftest import get_patched_exchange, log_has def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 49a6370bb..cb225e465 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -3,19 +3,20 @@ import json import math import random -import pytest from copy import deepcopy from typing import List from unittest.mock import MagicMock import numpy as np import pandas as pd +import pytest from arrow import Arrow -from freqtrade import optimize, constants, DependencyException +from freqtrade import DependencyException, constants, optimize from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange -from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration +from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, + start) from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 624f8ed77..4ab32343a 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -3,16 +3,19 @@ import json import os import uuid -import arrow from shutil import copyfile +import arrow + from freqtrade import optimize -from freqtrade.misc import file_dump_json -from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ - download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \ - load_cached_data_for_updating from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.misc import file_dump_json +from freqtrade.optimize.__init__ import (download_backtesting_testdata, + download_pairs, + load_cached_data_for_updating, + load_tickerdata_file, + make_testdata_path, trim_tickerlist) +from freqtrade.tests.conftest import get_patched_exchange, log_has # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 11db7ffb3..58514d1c0 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -13,7 +13,8 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC, RPCException from freqtrade.state import State -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap +from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, + patch_get_signal) # Functions for recurrent object patching diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 805424d26..5aea98d48 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -7,7 +7,7 @@ from copy import deepcopy from unittest.mock import MagicMock from freqtrade.rpc.rpc_manager import RPCManager -from freqtrade.tests.conftest import log_has, get_patched_freqtradebot +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has def test_rpc_manager_object() -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b2cca9b9a..2710328bd 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -11,17 +11,18 @@ from datetime import datetime from random import randint from unittest.mock import MagicMock -from telegram import Update, Message, Chat +from telegram import Chat, Message, Update from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -from freqtrade.rpc.telegram import Telegram -from freqtrade.rpc.telegram import authorized_only +from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State -from freqtrade.tests.conftest import get_patched_freqtradebot, patch_exchange, log_has -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap +from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, + patch_exchange) +from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, + patch_get_signal) class DummyCls(Telegram): diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 608e6a4a1..094c166b8 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,8 +1,9 @@ # pragma pylint: disable=missing-docstring,C0103,protected-access -import freqtrade.tests.conftest as tt # test tools from unittest.mock import MagicMock +import freqtrade.tests.conftest as tt # test tools + # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine # perhaps try to anticipate that by using some python package diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 02225acca..e6108e8f8 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -12,9 +12,9 @@ import arrow from pandas import DataFrame from freqtrade.analyze import Analyze, SignalType -from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.tests.conftest import get_patched_exchange, log_has # Avoid to reinit the same object again and again _ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index a4096cc01..e64e1b486 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -4,18 +4,18 @@ Unit test file for configuration.py """ import json +from argparse import Namespace from copy import deepcopy from unittest.mock import MagicMock -from argparse import Namespace import pytest from jsonschema import ValidationError +from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.constants import DEFAULT_DB_PROD_URL, DEFAULT_DB_DRYRUN_URL +from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has -from freqtrade import OperationalException def test_configuration_object() -> None: diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 2fb9219ca..5af85d268 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -5,7 +5,6 @@ import time from unittest.mock import MagicMock import pytest - from requests.exceptions import RequestException from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index dde2dd6d9..17bd6aa7c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -14,11 +14,13 @@ import arrow import pytest import requests -from freqtrade import constants, DependencyException, OperationalException, TemporaryError +from freqtrade import (DependencyException, OperationalException, + TemporaryError, constants) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.state import State -from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange +from freqtrade.tests.conftest import (log_has, patch_coinmarketcap, + patch_exchange) # Functions for recurrent object patching diff --git a/freqtrade/tests/test_indicator_helpers.py b/freqtrade/tests/test_indicator_helpers.py index 87b085a0b..f3d34ec0b 100644 --- a/freqtrade/tests/test_indicator_helpers.py +++ b/freqtrade/tests/test_indicator_helpers.py @@ -1,6 +1,6 @@ import pandas as pd -from freqtrade.indicator_helpers import went_up, went_down +from freqtrade.indicator_helpers import went_down, went_up def test_went_up(): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 414dcdbe9..20a02eedc 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -11,7 +11,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, set_loggers, reconfigure +from freqtrade.main import main, reconfigure, set_loggers from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index b57900428..e2ba40dee 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -8,8 +8,8 @@ import datetime from unittest.mock import MagicMock from freqtrade.analyze import Analyze -from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, - common_datearray, file_dump_json, format_ms_time) +from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, + file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 12ae6579f..b24f2dd6c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest from sqlalchemy import create_engine -from freqtrade import constants, OperationalException -from freqtrade.persistence import Trade, init, clean_dry_run_db +from freqtrade import OperationalException, constants +from freqtrade.persistence import Trade, clean_dry_run_db, init from freqtrade.tests.conftest import log_has From ac20bf31dfab8589f3f3ae8b09fb3fa8fa83a685 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 4 Jul 2018 14:23:06 +0200 Subject: [PATCH 230/239] Update ccxt from 1.15.7 to 1.15.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 98721b74c..8ab13d394 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.7 +ccxt==1.15.8 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From bfd1e90154b4db42fac2e867f381f9dfba6c825a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 5 Jul 2018 14:23:11 +0200 Subject: [PATCH 231/239] Update ccxt from 1.15.8 to 1.15.13 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ca1575d7a..c2303f11d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.8 +ccxt==1.15.13 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 239f8606e15fe0fe02d3b02db1fd2bbf28f1171e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 5 Jul 2018 14:23:12 +0200 Subject: [PATCH 232/239] Update pytest from 3.6.2 to 3.6.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c2303f11d..f87241e32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.14.5 TA-Lib==0.4.17 -pytest==3.6.2 +pytest==3.6.3 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 03c112a6010c07941c55d565e4be0fa47e00b2b6 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 14 Jun 2018 08:31:29 +0300 Subject: [PATCH 233/239] config, optimize: fstrings in use --- freqtrade/configuration.py | 6 +++--- freqtrade/optimize/__init__.py | 7 ++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index c90eddeb8..582b2889c 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -62,8 +62,8 @@ class Configuration(object): conf = json.load(file) except FileNotFoundError: raise OperationalException( - 'Config file "{}" not found!' - ' Please create a config file or check whether it exists.'.format(path)) + f'Config file "{path}" not found!' + ' Please create a config file or check whether it exists.') if 'internals' not in conf: conf['internals'] = {} @@ -109,7 +109,7 @@ class Configuration(object): config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') - logger.info('Using DB: "{}"'.format(config['db_url'])) + logger.info(f'Using DB: "{config["db_url"]}"') # Check if the exchange set by the user is supported self.check_exchange(config) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 1e76808e7..e806ff2b8 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -54,11 +54,8 @@ def load_tickerdata_file( :return dict OR empty if unsuccesful """ path = make_testdata_path(datadir) - pair_file_string = pair.replace('/', '_') - file = os.path.join(path, '{pair}-{ticker_interval}.json'.format( - pair=pair_file_string, - ticker_interval=ticker_interval, - )) + pair_s = pair.replace('/', '_') + file = os.path.join(path, f'{pair_s}-{ticker_interval}.json') gzipfile = file + '.gz' # If the file does not exist we download it when None is returned. From 7dca3c6d03e83fe512d85280ceab0a5c41673d80 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 14 Jun 2018 08:52:13 +0300 Subject: [PATCH 234/239] freqtradebot,main,hyperopt: fstrings in use --- freqtrade/freqtradebot.py | 8 ++------ freqtrade/main.py | 5 +---- freqtrade/optimize/hyperopt.py | 30 +++++++++++++++--------------- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 64ff170fd..9def7078c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -626,12 +626,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well else: - message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade - ) - + gain = "profit" if fmt_exp_profit > 0 else "loss" + message += f'` ({gain}: {fmt_exp_profit:.2f}%, {profit_trade:.8f})`' # Send the message self.rpc.send_msg(message) Trade.session.flush() diff --git a/freqtrade/main.py b/freqtrade/main.py index 9d17a403a..79080ce37 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -74,10 +74,7 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: # Create new instance freqtrade = FreqtradeBot(Configuration(args).get_config()) freqtrade.rpc.send_msg( - '*Status:* `Config reloaded ...`'.format( - freqtrade.state.name.lower() - ) - ) + '*Status:* `Config reloaded {freqtrade.state.name.lower()}...`') return freqtrade diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7138e2601..72bf34eb3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -128,13 +128,12 @@ class Hyperopt(Backtesting): Log results if it is better than any previous evaluation """ if results['loss'] < self.current_best_loss: + current = results['current_tries'] + total = results['total_tries'] + res = results['result'] + loss = results['loss'] self.current_best_loss = results['loss'] - log_msg = '\n{:5d}/{}: {}. Loss {:.5f}'.format( - results['current_tries'], - results['total_tries'], - results['result'], - results['loss'] - ) + log_msg = f'\n{current:5d}/{total}: {res}. Loss {loss:.5f}' print(log_msg) else: print('.', end='') @@ -309,15 +308,16 @@ class Hyperopt(Backtesting): """ Return the format result in a string """ - return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_abs.sum(), - self.config['stake_currency'], - results.profit_percent.sum(), - results.trade_duration.mean(), - ) + trades = len(results.index) + avg_profit = results.profit_percent.mean() * 100.0 + total_profit = results.profit_abs.sum() + stake_cur = self.config['stake_currency'] + profit = results.profit_percent.sum() + duration = results.trade_duration.mean() + + return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. ' + f'Total profit {total_profit: 11.8f} {stake_cur} ' + f'({profit:.4f}Σ%). Avg duration {duration:5.1f} mins.') def get_optimizer(self, cpu_count) -> Optimizer: return Optimizer( From a2063ede558a28e68e94fa385fa56416c20f8341 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 08:27:29 -0500 Subject: [PATCH 235/239] persistence: fstrings in use --- freqtrade/persistence.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 764ba6509..0e0b22e82 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -21,6 +21,7 @@ from freqtrade import OperationalException logger = logging.getLogger(__name__) _DECL_BASE: Any = declarative_base() +_SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' def init(config: Dict) -> None: @@ -45,10 +46,8 @@ def init(config: Dict) -> None: try: engine = create_engine(db_url, **kwargs) except NoSuchModuleError: - error = 'Given value for db_url: \'{}\' is no valid database URL! (See {}).'.format( - db_url, 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' - ) - raise OperationalException(error) + raise OperationalException(f'Given value for db_url: \'{db_url}\' ' + f'is no valid database URL! (See {_SQL_DOCS_URL})') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() @@ -173,13 +172,10 @@ class Trade(_DECL_BASE): max_rate = Column(Float, nullable=True, default=0.0) def __repr__(self): - return 'Trade(id={}, pair={}, amount={:.8f}, open_rate={:.8f}, open_since={})'.format( - self.id, - self.pair, - self.amount, - self.open_rate, - arrow.get(self.open_date).humanize() if self.is_open else 'closed' - ) + open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed' + + return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' + f'open_rate={self.open_rate:.8f}, open_since={open_since})') def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): """this adjusts the stop loss to it's most recently observed setting""" @@ -226,6 +222,7 @@ class Trade(_DECL_BASE): :param order: order retrieved by exchange.get_order() :return: None """ + order_type = order['type'] # Ignore open and cancelled orders if order['status'] == 'open' or order['price'] is None: return @@ -233,16 +230,16 @@ class Trade(_DECL_BASE): logger.info('Updating trade (id=%d) ...', self.id) getcontext().prec = 8 # Bittrex do not go above 8 decimal - if order['type'] == 'limit' and order['side'] == 'buy': + if order_type == 'limit' and order['side'] == 'buy': # Update open rate and actual amount self.open_rate = Decimal(order['price']) self.amount = Decimal(order['amount']) logger.info('LIMIT_BUY has been fulfilled for %s.', self) self.open_order_id = None - elif order['type'] == 'limit' and order['side'] == 'sell': + elif order_type == 'limit' and order['side'] == 'sell': self.close(order['price']) else: - raise ValueError('Unknown order type: {}'.format(order['type'])) + raise ValueError(f'Unknown order type: {order_type}') cleanup() def close(self, rate: float) -> None: @@ -313,7 +310,8 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - return float("{0:.8f}".format(close_trade_price - open_trade_price)) + profit = close_trade_price - open_trade_price + return float(f"{profit:.8f}") def calc_profit_percent( self, @@ -333,5 +331,5 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - - return float("{0:.8f}".format((close_trade_price / open_trade_price) - 1)) + profit_percent = (close_trade_price / open_trade_price) - 1 + return float(f"{profit_percent:.8f}") From 21fc933678ebf938e99fdcaa19f48d64844b26e9 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 16:35:35 -0500 Subject: [PATCH 236/239] convert_backtesting: fstrings in use --- scripts/convert_backtestdata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 698c1c829..96e0cbce8 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -143,15 +143,14 @@ def convert_main(args: Namespace) -> None: interval = str_interval break # change order on pairs if old ticker interval found + filename_new = path.join(path.dirname(filename), - "{}_{}-{}.json".format(currencies[1], - currencies[0], interval)) + f"{currencies[1]}_{currencies[0]}-{interval}.json") elif ret_string: interval = ret_string.group(0) filename_new = path.join(path.dirname(filename), - "{}_{}-{}.json".format(currencies[0], - currencies[1], interval)) + f"{currencies[0]}_{currencies[1]}-{interval}.json") else: logger.warning("file %s could not be converted, interval not found", filename) From adbffc69e152a9a9b4413a9b9c3f9e20baa3796b Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 17:17:10 -0500 Subject: [PATCH 237/239] telegram: fstrings in use --- freqtrade/rpc/telegram.py | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4dd23971b..13a2b1913 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -153,7 +153,7 @@ class Telegram(RPC): try: df_statuses = self._rpc_status_table() message = tabulate(df_statuses, headers='keys', tablefmt='simple') - self._send_msg("
{}
".format(message), parse_mode=ParseMode.HTML) + self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e), bot=bot) @@ -166,6 +166,8 @@ class Telegram(RPC): :param update: message update :return: None """ + stake_cur = self._config['stake_currency'] + fiat_disp_cur = self._config['fiat_display_currency'] try: timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): @@ -173,18 +175,17 @@ class Telegram(RPC): try: stats = self._rpc_daily_profit( timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] + stake_cur, + fiat_disp_cur ) stats = tabulate(stats, headers=[ 'Day', - 'Profit {}'.format(self._config['stake_currency']), - 'Profit {}'.format(self._config['fiat_display_currency']) + f'Profit {stake_cur}', + f'Profit {fiat_disp_cur}' ], tablefmt='simple') - message = 'Daily Profit over the last {} days:\n
{}
'\ - .format(timescale, stats) + message = f'Daily Profit over the last {timescale} days:\n
{stats}
' self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e), bot=bot) @@ -198,39 +199,38 @@ class Telegram(RPC): :param update: message update :return: None """ + stake_cur = self._config['stake_currency'] + fiat_disp_cur = self._config['fiat_display_currency'] + try: stats = self._rpc_trade_statistics( - self._config['stake_currency'], - self._config['fiat_display_currency']) - + stake_cur, + fiat_disp_cur) + profit_closed_coin = stats['profit_closed_coin'] + profit_closed_percent = stats['profit_closed_percent'] + profit_closed_fiat = stats['profit_closed_fiat'] + profit_all_coin = stats['profit_all_coin'] + profit_all_percent = stats['profit_all_percent'] + profit_all_fiat = stats['profit_all_fiat'] + trade_count = stats['trade_count'] + first_trade_date = stats['first_trade_date'] + latest_trade_date = stats['latest_trade_date'] + avg_duration = stats['avg_duration'] + best_pair = stats['best_pair'] + best_rate = stats['best_rate'] # Message to display markdown_msg = "*ROI:* Close trades\n" \ - "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ - "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ - "*ROI:* All trades\n" \ - "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ - "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ - "*Total Trade Count:* `{trade_count}`\n" \ - "*First Trade opened:* `{first_trade_date}`\n" \ - "*Latest Trade opened:* `{latest_trade_date}`\n" \ - "*Avg. Duration:* `{avg_duration}`\n" \ - "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ - .format( - coin=self._config['stake_currency'], - fiat=self._config['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) + f"∙ `{profit_closed_coin:.8f} {stake_cur} "\ + f"({profit_closed_percent:.2f}%)`\n" \ + f"∙ `{profit_closed_fiat:.3f} {fiat_disp_cur}`\n" \ + f"*ROI:* All trades\n" \ + f"∙ `{profit_all_coin:.8f} {stake_cur} ({profit_all_percent:.2f}%)`\n" \ + f"∙ `{profit_all_fiat:.3f} {fiat_disp_cur}`\n" \ + f"*Total Trade Count:* `{trade_count}`\n" \ + f"*First Trade opened:* `{first_trade_date}`\n" \ + f"*Latest Trade opened:* `{latest_trade_date}`\n" \ + f"*Avg. Duration:* `{avg_duration}`\n" \ + f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`" self._send_msg(markdown_msg, bot=bot) except RPCException as e: self._send_msg(str(e), bot=bot) From df68b0990ffc991032eadaabdf04d22f8db9ca4f Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Wed, 4 Jul 2018 13:53:45 -0500 Subject: [PATCH 238/239] rpc: fstrings --- freqtrade/rpc/rpc.py | 60 ++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f8cb136e4..263f293c6 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -74,34 +74,33 @@ class RPC(object): # calculate profit and send message to user current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] current_profit = trade.calc_profit_percent(current_rate) - fmt_close_profit = '{:.2f}%'.format( - round(trade.close_profit * 100, 2) - ) if trade.close_profit else None - message = "*Trade ID:* `{trade_id}`\n" \ - "*Current Pair:* [{pair}]({market_url})\n" \ - "*Open Since:* `{date}`\n" \ - "*Amount:* `{amount}`\n" \ - "*Open Rate:* `{open_rate:.8f}`\n" \ - "*Close Rate:* `{close_rate}`\n" \ - "*Current Rate:* `{current_rate:.8f}`\n" \ - "*Close Profit:* `{close_profit}`\n" \ - "*Current Profit:* `{current_profit:.2f}%`\n" \ - "*Open Order:* `{open_order}`"\ - .format( - trade_id=trade.id, - pair=trade.pair, - market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit=fmt_close_profit, - current_profit=round(current_profit * 100, 2), - open_order='({} {} rem={:.8f})'.format( - order['type'], order['side'], order['remaining'] - ) if order else None, - ) + fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' + if trade.close_profit else None) + market_url = self._freqtrade.exchange.get_pair_detail_url(trade.pair) + trade_date = arrow.get(trade.open_date).humanize() + open_rate = trade.open_rate + close_rate = trade.close_rate + amount = round(trade.amount, 8) + current_profit = round(current_profit * 100, 2) + if order: + order_type = order['type'] + order_side = order['side'] + order_rem = order['remaining'] + open_order = f'({order_type} {order_side} rem={order_rem:.8f})' + else: + open_order = None + + message = f"*Trade ID:* `{trade.id}`\n" \ + f"*Current Pair:* [{trade.pair}]({market_url})\n" \ + f"*Open Since:* `{trade_date}`\n" \ + f"*Amount:* `{amount}`\n" \ + f"*Open Rate:* `{open_rate:.8f}`\n" \ + f"*Close Rate:* `{close_rate}`\n" \ + f"*Current Rate:* `{current_rate:.8f}`\n" \ + f"*Close Profit:* `{fmt_close_profit}`\n" \ + f"*Current Profit:* `{current_profit:.2f}%`\n" \ + f"*Open Order:* `{open_order}`"\ + result.append(message) return result @@ -116,11 +115,12 @@ class RPC(object): for trade in trades: # calculate profit and send message to user current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + trade_perc = (100 * trade.calc_profit_percent(current_rate)) trades_list.append([ trade.id, trade.pair, shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) + f'{trade_perc:.2f}%' ]) columns = ['ID', 'Pair', 'Since', 'Profit'] @@ -148,7 +148,7 @@ class RPC(object): .all() curdayprofit = sum(trade.calc_profit() for trade in trades) profit_days[profitday] = { - 'amount': format(curdayprofit, '.8f'), + 'amount': f'{curdayprofit:.8f}', 'trades': len(trades) } From e808b3a2a1c73a1f30d33d073cf2245013d46d0e Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 5 Jul 2018 10:47:08 -0500 Subject: [PATCH 239/239] rpc: get rid of extra else and fix mypy warning --- freqtrade/rpc/rpc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 263f293c6..11658c6fb 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -82,13 +82,12 @@ class RPC(object): close_rate = trade.close_rate amount = round(trade.amount, 8) current_profit = round(current_profit * 100, 2) + open_order = '' if order: order_type = order['type'] order_side = order['side'] order_rem = order['remaining'] open_order = f'({order_type} {order_side} rem={order_rem:.8f})' - else: - open_order = None message = f"*Trade ID:* `{trade.id}`\n" \ f"*Current Pair:* [{trade.pair}]({market_url})\n" \