From 5db883906af22eeb086174d1fcfae149f6978ed4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 13 Dec 2019 06:52:33 +0100 Subject: [PATCH 001/128] Try to verify available amount on the exchange --- freqtrade/freqtradebot.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0595e0d35..ac73f6d65 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -889,6 +889,27 @@ class FreqtradeBot: # TODO: figure out how to handle partially complete sell orders return False + def _safe_sell_amount(self, pair: str, amount: float) -> float: + """ + Get sellable amount. + Should be trade.amount - but will fall back to the available amount if necessary. + This should cover cases where get_real_amount() was not able to update the amount + for whatever reason. + :param pair: pair - used for logging + :param amount: amount we expect to be available + :return: amount to sell + :raise: DependencyException: if available balance is not within 2% of the available amount. + """ + wallet_amount = self.wallets.get_free(pair) + logger.info(f"Amounts: {wallet_amount} - {amount}") + if wallet_amount > amount: + return amount + elif wallet_amount > amount * 0.98: + logger.info(f"{pair} - Falling back to wallet-amount.") + return wallet_amount + else: + raise DependencyException("Not enough amount to sell.") + def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> None: """ Executes a limit sell for the given trade and limit @@ -919,10 +940,12 @@ class FreqtradeBot: # Emergencysells (default to market!) ordertype = self.strategy.order_types.get("emergencysell", "market") + amount = self._safe_sell_amount(trade.pair, trade.amount) + # Execute sell and update trade record order = self.exchange.sell(pair=str(trade.pair), ordertype=ordertype, - amount=trade.amount, rate=limit, + amount=amount, rate=limit, time_in_force=self.strategy.order_time_in_force['sell'] ) From 04257d8ecc357dea23face544f8e979834a20e19 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 13 Dec 2019 07:06:54 +0100 Subject: [PATCH 002/128] Add tests for safe_sell_amount --- freqtrade/freqtradebot.py | 2 +- tests/test_freqtradebot.py | 76 +++++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ac73f6d65..3ebe89a71 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -901,7 +901,7 @@ class FreqtradeBot: :raise: DependencyException: if available balance is not within 2% of the available amount. """ wallet_amount = self.wallets.get_free(pair) - logger.info(f"Amounts: {wallet_amount} - {amount}") + logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}") if wallet_amount > amount: return amount elif wallet_amount > amount * 0.98: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index efab64a6a..9c2fd9ddc 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -883,7 +883,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: 'freqtrade.freqtradebot.FreqtradeBot', get_target_bid=get_bid, _get_min_pair_stake_amount=MagicMock(return_value=1) - ) + ) buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -2314,6 +2314,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException()) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=300)) sellmock = MagicMock() patch_exchange(mocker) mocker.patch.multiple( @@ -2591,7 +2592,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple( - sell_flag=False, sell_type=SellType.NONE)) + sell_flag=False, sell_type=SellType.NONE)) freqtrade.create_trades() trade = Trade.query.first() @@ -2631,6 +2632,77 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke assert trade.sell_reason == SellType.SELL_SIGNAL.value +def test_sell_not_enough_balance(default_conf, limit_buy_order, + fee, mocker, caplog) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.00002172, + 'ask': 0.00002173, + 'last': 0.00002172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) + + freqtrade.create_trades() + + trade = Trade.query.first() + amnt = trade.amount + trade.update(limit_buy_order) + patch_get_signal(freqtrade, value=(False, True)) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=trade.amount * 0.985)) + + assert freqtrade.handle_trade(trade) is True + assert log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert trade.amount != amnt + + +def test__safe_sell_amount(default_conf, caplog, mocker): + patch_RPCManager(mocker) + patch_exchange(mocker) + amount = 95.33 + amount_wallet = 95.29 + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + assert freqtrade._safe_sell_amount(trade.pair, trade.amount) == amount_wallet + assert log_has_re(r'.*Falling back to wallet-amount.', caplog) + + +def test__safe_sell_amount_error(default_conf, caplog, mocker): + patch_RPCManager(mocker) + patch_exchange(mocker) + amount = 95.33 + amount_wallet = 91.29 + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + with pytest.raises(DependencyException, match=r"Not enough amount to sell."): + assert freqtrade._safe_sell_amount(trade.pair, trade.amount) + + def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 931d24b5a8c028b879518921c0c67f79865b9dcf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 09:22:15 +0100 Subject: [PATCH 003/128] Have dry_run_wallet default to 1000 --- docs/configuration.md | 2 +- freqtrade/constants.py | 5 +++-- freqtrade/exchange/exchange.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5ad1a886e..927432a46 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,7 +47,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `ticker_interval` | The ticker interval to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
***Datatype:*** *String* | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `dry_run_wallet` | Overrides the default amount of 999.9 stake currency units in the wallet used by the bot running in the Dry Run mode if you need it for any reason.
***Datatype:*** *Float* +| `dry_run_wallet` | Overrides the default amount of 1000 stake currency units in the wallet used by the bot running in the Dry Run mode if you need it for any reason.
***Datatype:*** *Float* | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* | `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* | `stoploss` | **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float (as ratio)* diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f5e5969eb..5c7190b41 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -18,7 +18,7 @@ REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter'] -DRY_RUN_WALLET = 999.9 +DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons USERPATH_HYPEROPTS = 'hyperopts' @@ -75,7 +75,7 @@ CONF_SCHEMA = { }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, - 'dry_run_wallet': {'type': 'number'}, + 'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET}, 'process_only_new_candles': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', @@ -275,6 +275,7 @@ CONF_SCHEMA = { 'stake_currency', 'stake_amount', 'dry_run', + 'dry_run_wallet', 'bid_strategy', 'unfilledtimeout', 'stoploss', diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index fbe7cd29a..a148f9dae 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -479,7 +479,7 @@ class Exchange: @retrier def get_balance(self, currency: str) -> float: if self._config['dry_run']: - return constants.DRY_RUN_WALLET + return self._config['dry_run_wallet'] # ccxt exception is already handled by get_balances balances = self.get_balances() From 52b212db64ae6d679f76f6e9c0af382f54b13751 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 09:37:08 +0100 Subject: [PATCH 004/128] Fix tests after changing dry_run_wallet amount --- tests/exchange/test_exchange.py | 1 + tests/test_configuration.py | 7 ++++--- tests/test_freqtradebot.py | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a21a5f3ac..774ad8cf2 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -876,6 +876,7 @@ def test_sell_considers_time_in_force(default_conf, mocker, exchange_name): def test_get_balance_dry_run(default_conf, mocker): default_conf['dry_run'] = True + default_conf['dry_run_wallet'] = 999.9 exchange = get_patched_exchange(mocker, default_conf) assert exchange.get_balance(currency='BTC') == 999.9 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 89ca74afa..292d53315 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -8,9 +8,9 @@ from pathlib import Path from unittest.mock import MagicMock import pytest -from jsonschema import Draft4Validator, ValidationError, validate +from jsonschema import ValidationError -from freqtrade import OperationalException, constants +from freqtrade import OperationalException from freqtrade.configuration import (Arguments, Configuration, check_exchange, remove_credentials, validate_config_consistency) @@ -718,7 +718,8 @@ def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None: def test_validate_default_conf(default_conf) -> None: - validate(default_conf, constants.CONF_SCHEMA, Draft4Validator) + # Validate via our validator - we allow setting defaults! + validate_config_schema(default_conf) def test_validate_tsl(default_conf): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5e197da71..a73fd6c61 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -211,6 +211,7 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) + edge_conf['dry_run_wallet'] = 999.9 freqtrade = FreqtradeBot(edge_conf) assert freqtrade._get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20 @@ -1338,6 +1339,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, patch_exchange(mocker) patch_edge(mocker) edge_conf['max_open_trades'] = float('inf') + edge_conf['dry_run_wallet'] = 999.9 mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock(return_value={ From fda8f7e30599810a834e2442a484d714e2ba3463 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 09:38:18 +0100 Subject: [PATCH 005/128] Introuce WalletDry - supporting dry-run wallets --- freqtrade/freqtradebot.py | 13 ++++++++----- freqtrade/rpc/telegram.py | 6 ++++++ freqtrade/wallets.py | 41 ++++++++++++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0595e0d35..df9fd0b17 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -25,7 +25,7 @@ from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType -from freqtrade.wallets import Wallets +from freqtrade.wallets import Wallets, WalletsDry logger = logging.getLogger(__name__) @@ -62,7 +62,13 @@ class FreqtradeBot: self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange - self.wallets = Wallets(self.config, self.exchange) + persistence.init(self.config.get('db_url', None), + clean_open_orders=self.config.get('dry_run', False)) + + if self.config['dry_run']: + self.wallets = WalletsDry(self.config, self.exchange) + else: + self.wallets = Wallets(self.config, self.exchange) self.dataprovider = DataProvider(self.config, self.exchange) # Attach Dataprovider to Strategy baseclass @@ -78,9 +84,6 @@ class FreqtradeBot: self.active_pair_whitelist = self._refresh_whitelist() - persistence.init(self.config.get('db_url', None), - clean_open_orders=self.config.get('dry_run', False)) - # Set initial bot state from config initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2e736f11a..4d7857f44 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -331,7 +331,13 @@ class Telegram(RPC): try: result = self._rpc_balance(self._config['stake_currency'], self._config.get('fiat_display_currency', '')) + output = '' + if self._config['dry_run']: + output += ( + f"Simulated balances - starting capital: " + f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" + ) for currency in result['currencies']: if currency['est_stake'] > 0.0001: curr_output = "*{currency}:*\n" \ diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index c674b5286..eb2603776 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -28,9 +28,6 @@ class Wallets: def get_free(self, currency) -> float: - if self._config['dry_run']: - return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET) - balance = self._wallets.get(currency) if balance and balance.free: return balance.free @@ -39,9 +36,6 @@ class Wallets: def get_used(self, currency) -> float: - if self._config['dry_run']: - return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET) - balance = self._wallets.get(currency) if balance and balance.used: return balance.used @@ -50,9 +44,6 @@ class Wallets: def get_total(self, currency) -> float: - if self._config['dry_run']: - return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET) - balance = self._wallets.get(currency) if balance and balance.total: return balance.total @@ -75,3 +66,35 @@ class Wallets: def get_all_balances(self) -> Dict[str, Any]: return self._wallets + + +class WalletsDry(Wallets): + + def __init__(self, config: dict, exchange: Exchange) -> None: + self.start_cap = config['dry_run_wallet'] + super().__init__(config, exchange) + + def update(self) -> None: + """ Update does not do anything in dry-mode...""" + from freqtrade.persistence import Trade + closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all() + print(len(closed_trades)) + tot_profit = sum([trade.calc_profit() for trade in closed_trades]) + current_stake = self.start_cap + tot_profit + self._wallets[self._config['stake_currency']] = Wallet( + self._config['stake_currency'], + current_stake, + 0, + current_stake + ) + open_trades = Trade.get_trades(Trade.is_open.is_(True)).all() + + for trade in open_trades: + curr = trade.pair.split('/')[0] + trade.amount + self._wallets[curr] = Wallet( + curr, + trade.amount, + 0, + trade.amount + ) From f0bbc75038134d7debb9aed56d85eb2c2702c6b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 09:48:35 +0100 Subject: [PATCH 006/128] Combine dry_run wallet into original Wallets class --- freqtrade/freqtradebot.py | 8 ++--- freqtrade/rpc/telegram.py | 2 +- freqtrade/wallets.py | 64 +++++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index df9fd0b17..a242c11ae 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -25,7 +25,7 @@ from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType -from freqtrade.wallets import Wallets, WalletsDry +from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) @@ -65,10 +65,8 @@ class FreqtradeBot: persistence.init(self.config.get('db_url', None), clean_open_orders=self.config.get('dry_run', False)) - if self.config['dry_run']: - self.wallets = WalletsDry(self.config, self.exchange) - else: - self.wallets = Wallets(self.config, self.exchange) + self.wallets = Wallets(self.config, self.exchange) + self.dataprovider = DataProvider(self.config, self.exchange) # Attach Dataprovider to Strategy baseclass diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4d7857f44..e36b46ba7 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -335,7 +335,7 @@ class Telegram(RPC): output = '' if self._config['dry_run']: output += ( - f"Simulated balances - starting capital: " + f"*Warning:*Simulated balances in Dry Mode.\nStarting capital: " f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" ) for currency in result['currencies']: diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index eb2603776..f8dd0ee2f 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -4,7 +4,7 @@ import logging from typing import Dict, NamedTuple, Any from freqtrade.exchange import Exchange -from freqtrade import constants +from freqtrade.persistence import Trade logger = logging.getLogger(__name__) @@ -23,6 +23,7 @@ class Wallets: self._config = config self._exchange = exchange self._wallets: Dict[str, Wallet] = {} + self.start_cap = config['dry_run_wallet'] self.update() @@ -50,36 +51,12 @@ class Wallets: else: return 0 - def update(self) -> None: - - balances = self._exchange.get_balances() - - for currency in balances: - self._wallets[currency] = Wallet( - currency, - balances[currency].get('free', None), - balances[currency].get('used', None), - balances[currency].get('total', None) - ) - - logger.info('Wallets synced.') - - def get_all_balances(self) -> Dict[str, Any]: - return self._wallets - - -class WalletsDry(Wallets): - - def __init__(self, config: dict, exchange: Exchange) -> None: - self.start_cap = config['dry_run_wallet'] - super().__init__(config, exchange) - - def update(self) -> None: - """ Update does not do anything in dry-mode...""" - from freqtrade.persistence import Trade + def _update_dry(self) -> None: + """ Update from database in dry-run mode""" closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all() - print(len(closed_trades)) + tot_profit = sum([trade.calc_profit() for trade in closed_trades]) + current_stake = self.start_cap + tot_profit self._wallets[self._config['stake_currency']] = Wallet( self._config['stake_currency'], @@ -98,3 +75,32 @@ class WalletsDry(Wallets): 0, trade.amount ) + + def _update_live(self) -> None: + + balances = self._exchange.get_balances() + + for currency in balances: + self._wallets[currency] = Wallet( + currency, + balances[currency].get('free', None), + balances[currency].get('used', None), + balances[currency].get('total', None) + ) + + def update(self) -> None: + if self._config['dry_run']: + self._update_dry() + else: + self._update_live() + logger.info('Wallets synced.') + + def get_all_balances(self) -> Dict[str, Any]: + return self._wallets + + +class WalletsDry(Wallets): + + def __init__(self, config: dict, exchange: Exchange) -> None: + + super().__init__(config, exchange) From 4463d58470aff647fe3769e52c2c90c12588ca98 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 09:49:56 +0100 Subject: [PATCH 007/128] Add release section about collapsible section --- docs/developer.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index d731f1768..fe37c140e 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -246,6 +246,17 @@ Determine if crucial bugfixes have been made between this commit and the current git log --oneline --no-decorate --no-merges master..new_release ``` +To keep the release-log short, best wrap the full git changelog into a collapsible details secction. + +```markdown +
+Expand full changelog + +... Full git changelog + +
+``` + ### Create github release / tag Once the PR against master is merged (best right after merging): From 5a5741878cecbadae067c8ee0c29b211161a3aeb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 10:26:56 +0100 Subject: [PATCH 008/128] Improve dry-run calculations --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/wallets.py | 7 +++---- tests/test_freqtradebot.py | 30 ++++++++++++++++++++++++++++++ tests/test_integration.py | 1 + 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a242c11ae..5c3ef64b1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -232,8 +232,8 @@ class FreqtradeBot: # Check if stake_amount is fulfilled if available_amount < stake_amount: raise DependencyException( - f"Available balance({available_amount} {self.config['stake_currency']}) is " - f"lower than stake amount({stake_amount} {self.config['stake_currency']})" + f"Available balance ({available_amount} {self.config['stake_currency']}) is " + f"lower than stake amount ({stake_amount} {self.config['stake_currency']})" ) return stake_amount diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index f8dd0ee2f..9ee305aab 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -54,21 +54,20 @@ class Wallets: def _update_dry(self) -> None: """ Update from database in dry-run mode""" closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all() - + open_trades = Trade.get_trades(Trade.is_open.is_(True)).all() tot_profit = sum([trade.calc_profit() for trade in closed_trades]) + tot_in_trades = sum([trade.stake_amount for trade in open_trades]) - current_stake = self.start_cap + tot_profit + current_stake = self.start_cap + tot_profit - tot_in_trades self._wallets[self._config['stake_currency']] = Wallet( self._config['stake_currency'], current_stake, 0, current_stake ) - open_trades = Trade.get_trades(Trade.is_open.is_(True)).all() for trade in open_trades: curr = trade.pair.split('/')[0] - trade.amount self._wallets[curr] = Wallet( curr, trade.amount, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a73fd6c61..a60ea8c7c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3485,3 +3485,33 @@ def test_process_i_am_alive(default_conf, mocker, caplog): ftbot.process() assert log_has_re(message, caplog) + + +@pytest.mark.usefixtures("init_persistence") +def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order): + default_conf['dry_run'] = True + # Initialize to 2 times stake amount + default_conf['dry_run_wallet'] = 0.002 + default_conf['max_open_trades'] = 2 + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + bot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(bot) + assert bot.wallets.get_free('BTC') == 0.002 + + bot.create_trades() + trades = Trade.query.all() + assert len(trades) == 2 + + bot.config['max_open_trades'] = 3 + with pytest.raises( + DependencyException, + match=r"Available balance \(0 BTC\) is lower than stake amount \(0.001 BTC\)"): + bot.create_trades() + diff --git a/tests/test_integration.py b/tests/test_integration.py index 228ed8468..728e96d55 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -71,6 +71,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) + mocker.patch("freqtrade.wallets.Wallets.get_free", MagicMock(return_value=1)) freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True From c741b67c3c1be52872ec12273504af0a68c1975f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 10:39:52 +0100 Subject: [PATCH 009/128] Adjust tests for dry_run wallet simulation --- freqtrade/exchange/exchange.py | 2 +- tests/rpc/test_rpc.py | 2 +- tests/rpc/test_rpc_apiserver.py | 1 + tests/rpc/test_rpc_telegram.py | 6 ++++-- tests/test_freqtradebot.py | 1 - tests/test_wallets.py | 3 ++- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a148f9dae..2401c59b6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -18,7 +18,7 @@ from ccxt.base.decimal_to_precision import ROUND_DOWN, ROUND_UP from pandas import DataFrame from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError, constants) + OperationalException, TemporaryError) from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.misc import deep_merge_dicts diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 699f2d962..0a8c1cabd 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -398,7 +398,7 @@ def test_rpc_balance_handle(default_conf, mocker, tickers): get_valid_pair_combination=MagicMock( side_effect=lambda a, b: f"{b}/{a}" if a == "USDT" else f"{a}/{b}") ) - + default_conf['dry_run'] = False freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 555fcdc81..ebb70bdf8 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -230,6 +230,7 @@ def test_api_stopbuy(botclient): def test_api_balance(botclient, mocker, rpc_balance): ftbot, client = botclient + ftbot.config['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', side_effect=lambda a, b: f"{a}/{b}") diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 2ba1ccf4b..b02f11394 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -462,7 +462,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tickers) -> None: - + default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', @@ -494,6 +494,7 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick def test_balance_handle_empty_response(default_conf, update, mocker) -> None: + default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) msg_mock = MagicMock() @@ -533,7 +534,8 @@ def test_balance_handle_empty_response_dry(default_conf, update, mocker) -> None telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 - assert "Running in Dry Run, balances are not available." in result + assert "*Warning:*Simulated balances in Dry Mode." in result + assert "Starting capital: `1000` BTC" in result def test_balance_handle_too_large_response(default_conf, update, mocker) -> None: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a60ea8c7c..18f5a461a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3514,4 +3514,3 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order) DependencyException, match=r"Available balance \(0 BTC\) is lower than stake amount \(0.001 BTC\)"): bot.create_trades() - diff --git a/tests/test_wallets.py b/tests/test_wallets.py index ae2810a2d..3177edc05 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -1,7 +1,8 @@ # pragma pylint: disable=missing-docstring -from tests.conftest import get_patched_freqtradebot from unittest.mock import MagicMock +from tests.conftest import get_patched_freqtradebot + def test_sync_wallet_at_boot(mocker, default_conf): default_conf['dry_run'] = False From 23d467eb0d827f6abd6641b786fe1e2a72da2e9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 10:41:57 +0100 Subject: [PATCH 010/128] Show simulation note also in restserver --- freqtrade/rpc/rpc.py | 1 + freqtrade/rpc/telegram.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 4cebe646e..84b72fe18 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -348,6 +348,7 @@ class RPC: 'total': total, 'symbol': symbol, 'value': value, + 'note': 'Simulated balances' if self._freqtrade.config.get('dry_run', False) else '' } def _rpc_start(self) -> Dict[str, str]: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e36b46ba7..c1572bb39 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -335,7 +335,9 @@ class Telegram(RPC): output = '' if self._config['dry_run']: output += ( - f"*Warning:*Simulated balances in Dry Mode.\nStarting capital: " + f"*Warning:*Simulated balances in Dry Mode.\n" + "This mode is still experimental!\n" + "Starting capital: " f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" ) for currency in result['currencies']: From 56e13c8919319997e99835ba645c7c3c175779ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 10:55:15 +0100 Subject: [PATCH 011/128] Enhance documentation for dry-run wallet --- docs/configuration.md | 10 ++++++---- freqtrade/wallets.py | 7 ------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 927432a46..f0724abc4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,7 +47,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `ticker_interval` | The ticker interval to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
***Datatype:*** *String* | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `dry_run_wallet` | Overrides the default amount of 1000 stake currency units in the wallet used by the bot running in the Dry Run mode if you need it for any reason.
***Datatype:*** *Float* +| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.
*Defaults to `1000`.*
***Datatype:*** *Float* | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* | `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* | `stoploss` | **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float (as ratio)* @@ -498,8 +498,10 @@ creating trades on the exchange. } ``` -Once you will be happy with your bot performance running in the Dry-run mode, -you can switch it to production mode. +Once you will be happy with your bot performance running in the Dry-run mode, you can switch it to production mode. + +!!! Note + A simulated wallet is available during dry-run mode, and will assume a starting capital of `dry_run_wallet` (defaults to 1000). ## Switch to production mode @@ -529,7 +531,7 @@ you run it in production mode. ``` !!! Note - If you have an exchange API key yet, [see our tutorial](/pre-requisite). + If you have an exchange API key yet, [see our tutorial](installation.md#setup-your-exchange-account). You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 9ee305aab..4c3a0f657 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -96,10 +96,3 @@ class Wallets: def get_all_balances(self) -> Dict[str, Any]: return self._wallets - - -class WalletsDry(Wallets): - - def __init__(self, config: dict, exchange: Exchange) -> None: - - super().__init__(config, exchange) From b5b6458f128a13920e5b0d7bf3f1fe1b084ead22 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 10:57:27 +0100 Subject: [PATCH 012/128] Add note about unlimited stake amount --- docs/configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index f0724abc4..9490927df 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -149,6 +149,9 @@ In this case a trade amount is calculated as: currency_balance / (max_open_trades - current_open_trades) ``` +!!! Note "When using Dry-Run Mode" + When using `"stake_amount" : "unlimited",` in combination with Dry-Run, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time. It is therefore important to set `dry_run_wallet` to a sensible value, otherwise it may simulate trades with 100 BTC (or more) at once - which may not correspond to your real available balance. + ### Understand minimal_roi The `minimal_roi` configuration parameter is a JSON object where the key is a duration From ce845ab092e38448cf5f04443d36c004d4feb2c5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Dec 2019 11:03:40 +0100 Subject: [PATCH 013/128] Improve docstring for dry-run wallet method --- freqtrade/wallets.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 4c3a0f657..dd706438f 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -52,7 +52,12 @@ class Wallets: return 0 def _update_dry(self) -> None: - """ Update from database in dry-run mode""" + """ + Update from database in dry-run mode + - Apply apply profits of closed trades on top of stake amount + - Subtract currently tied up stake_amount in open trades + - update balances for currencies currently in trades + """ closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all() open_trades = Trade.get_trades(Trade.is_open.is_(True)).all() tot_profit = sum([trade.calc_profit() for trade in closed_trades]) From 655672c9575132ef5d143afd4ee38679e9d8b352 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2019 06:22:54 +0100 Subject: [PATCH 014/128] Enhance documentation Note --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9490927df..2c8f7cea7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -150,7 +150,7 @@ currency_balance / (max_open_trades - current_open_trades) ``` !!! Note "When using Dry-Run Mode" - When using `"stake_amount" : "unlimited",` in combination with Dry-Run, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time. It is therefore important to set `dry_run_wallet` to a sensible value, otherwise it may simulate trades with 100 BTC (or more) at once - which may not correspond to your real available balance. + When using `"stake_amount" : "unlimited",` in combination with Dry-Run, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time. It is therefore important to set `dry_run_wallet` to a sensible value (like 0.05 or 0.01 for BTC and 1000 or 100 for USDT, for example), otherwise it may simulate trades with 100 BTC (or more) or 0.05 USDT (or less) at once - which may not correspond to your real available balance or is less than the exchange minimal limit for the order amount for the stake currency. ### Understand minimal_roi From e398c37526e3817ea3d452db15098240c7f70e80 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 07:29:42 +0000 Subject: [PATCH 015/128] Bump pytest from 5.3.1 to 5.3.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.3.1 to 5.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.3.1...5.3.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f073ece6e..fe5b4e369 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ flake8==3.7.9 flake8-type-annotations==0.1.0 flake8-tidy-imports==3.1.0 mypy==0.750 -pytest==5.3.1 +pytest==5.3.2 pytest-asyncio==0.10.0 pytest-cov==2.8.1 pytest-mock==1.13.0 From 33db37a915ab2f2194483256194963f2021ec56d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 07:30:04 +0000 Subject: [PATCH 016/128] Bump joblib from 0.14.0 to 0.14.1 Bumps [joblib](https://github.com/joblib/joblib) from 0.14.0 to 0.14.1. - [Release notes](https://github.com/joblib/joblib/releases) - [Changelog](https://github.com/joblib/joblib/blob/master/CHANGES.rst) - [Commits](https://github.com/joblib/joblib/compare/0.14.0...0.14.1) Signed-off-by: dependabot-preview[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 2317cdf3e..b2428e37d 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,4 +6,4 @@ scipy==1.3.3 scikit-learn==0.22 scikit-optimize==0.5.2 filelock==3.0.12 -joblib==0.14.0 +joblib==0.14.1 From c05af1b63c57d5eb2014a2d800a6a6e72e48d506 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 07:30:27 +0000 Subject: [PATCH 017/128] Bump plotly from 4.3.0 to 4.4.1 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.3.0 to 4.4.1. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.3.0...v4.4.1) Signed-off-by: dependabot-preview[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 87d5553b6..415d4b888 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.3.0 +plotly==4.4.1 From cc41cdbf2277f8fd793317ba9af690ba9f957b9e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 07:30:46 +0000 Subject: [PATCH 018/128] Bump mkdocs-material from 4.5.1 to 4.6.0 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 4.5.1 to 4.6.0. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/4.5.1...4.6.0) Signed-off-by: dependabot-preview[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index ae77c0b06..3e53c15e3 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,2 @@ -mkdocs-material==4.5.1 +mkdocs-material==4.6.0 mdx_truly_sane_lists==1.2 From 05de60a7febdfe846baa6c0f8714a18dd9ab81cf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 07:31:38 +0000 Subject: [PATCH 019/128] Bump cachetools from 3.1.1 to 4.0.0 Bumps [cachetools](https://github.com/tkem/cachetools) from 3.1.1 to 4.0.0. - [Release notes](https://github.com/tkem/cachetools/releases) - [Changelog](https://github.com/tkem/cachetools/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tkem/cachetools/compare/v3.1.1...v4.0.0) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index f00425e5b..3371537b7 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -4,7 +4,7 @@ ccxt==1.20.46 SQLAlchemy==1.3.11 python-telegram-bot==12.2.0 arrow==0.15.4 -cachetools==3.1.1 +cachetools==4.0.0 requests==2.22.0 urllib3==1.25.7 wrapt==1.11.2 From 75e6acd6ed54624c25479a386ccb5ce559354d5e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 09:46:17 +0000 Subject: [PATCH 020/128] Bump ccxt from 1.20.46 to 1.20.84 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.20.46 to 1.20.84. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.20.46...1.20.84) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 3371537b7..a6d9a6f5e 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.20.46 +ccxt==1.20.84 SQLAlchemy==1.3.11 python-telegram-bot==12.2.0 arrow==0.15.4 From 2af9ffa7f2cf3f7bdfce4a4b582df667d5bd6cc5 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 21:43:33 +0300 Subject: [PATCH 021/128] Align refresh_backtest_ to each other --- freqtrade/data/history.py | 16 ++++++++-------- freqtrade/utils.py | 2 +- tests/data/test_history.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index ddcae89ab..03aaeb0cc 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -334,12 +334,12 @@ def download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], - dl_path: Path, timerange: Optional[TimeRange] = None, + datadir: Path, timerange: Optional[TimeRange] = None, erase=False) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. - Used by freqtrade download-data - :return: Pairs not available + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. """ pairs_not_available = [] for pair in pairs: @@ -349,14 +349,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes continue for timeframe in timeframes: - dl_file = pair_data_filename(dl_path, pair, timeframe) + dl_file = pair_data_filename(datadir, pair, timeframe) if erase and dl_file.exists(): logger.info( f'Deleting existing data for pair {pair}, interval {timeframe}.') dl_file.unlink() logger.info(f'Downloading pair {pair}, interval {timeframe}.') - download_pair_history(datadir=dl_path, exchange=exchange, + download_pair_history(datadir=datadir, exchange=exchange, pair=pair, timeframe=str(timeframe), timerange=timerange) return pairs_not_available @@ -407,9 +407,9 @@ def download_trades_history(datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, timerange: TimeRange, erase=False) -> List[str]: """ - Refresh stored trades data. - Used by freqtrade download-data - :return: Pairs not available + Refresh stored trades data for backtesting and hyperopt operations. + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. """ pairs_not_available = [] for pair in pairs: diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 230fcf268..9e01c7ea6 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -213,7 +213,7 @@ def start_download_data(args: Dict[str, Any]) -> None: else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], - dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) + datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 6d89ab7c5..586492703 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -580,7 +580,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test ex = get_patched_exchange(mocker, default_conf) timerange = TimeRange.parse_timerange("20190101-20190102") refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], - timeframes=["1m", "5m"], dl_path=testdatadir, + timeframes=["1m", "5m"], datadir=testdatadir, timerange=timerange, erase=True ) @@ -600,7 +600,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): timerange = TimeRange.parse_timerange("20190101-20190102") unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], timeframes=["1m", "5m"], - dl_path=testdatadir, + datadir=testdatadir, timerange=timerange, erase=False ) From 4cd45b6535d4ed613718c7c5656701a81891c1ee Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 21:57:03 +0300 Subject: [PATCH 022/128] Rename download_*_history as non-public --- freqtrade/data/history.py | 40 ++++++++++++++--------------- tests/data/test_history.py | 52 +++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 03aaeb0cc..15c5ed451 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -155,11 +155,11 @@ def load_pair_history(pair: str, # The user forced the refresh of pairs if refresh_pairs: - download_pair_history(datadir=datadir, - exchange=exchange, - pair=pair, - timeframe=timeframe, - timerange=timerange) + _download_pair_history(datadir=datadir, + exchange=exchange, + pair=pair, + timeframe=timeframe, + timerange=timerange) pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) @@ -277,11 +277,11 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, return (data, since_ms) -def download_pair_history(datadir: Path, - exchange: Optional[Exchange], - pair: str, - timeframe: str = '5m', - timerange: Optional[TimeRange] = None) -> bool: +def _download_pair_history(datadir: Path, + exchange: Optional[Exchange], + pair: str, + timeframe: str = '5m', + timerange: Optional[TimeRange] = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -356,16 +356,16 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes dl_file.unlink() logger.info(f'Downloading pair {pair}, interval {timeframe}.') - download_pair_history(datadir=datadir, exchange=exchange, - pair=pair, timeframe=str(timeframe), - timerange=timerange) + _download_pair_history(datadir=datadir, exchange=exchange, + pair=pair, timeframe=str(timeframe), + timerange=timerange) return pairs_not_available -def download_trades_history(datadir: Path, - exchange: Exchange, - pair: str, - timerange: Optional[TimeRange] = None) -> bool: +def _download_trades_history(datadir: Path, + exchange: Exchange, + pair: str, + timerange: Optional[TimeRange] = None) -> bool: """ Download trade history from the exchange. Appends to previously downloaded trades data. @@ -425,9 +425,9 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: dl_file.unlink() logger.info(f'Downloading trades for pair {pair}.') - download_trades_history(datadir=datadir, exchange=exchange, - pair=pair, - timerange=timerange) + _download_trades_history(datadir=datadir, exchange=exchange, + pair=pair, + timerange=timerange) return pairs_not_available diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 586492703..c304aa97d 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -14,9 +14,9 @@ from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.history import (_load_cached_data_for_updating, + _download_pair_history, + _download_trades_history, convert_trades_to_ohlcv, - download_pair_history, - download_trades_history, load_tickerdata_file, pair_data_filename, pair_trades_filename, refresh_backtest_ohlcv_data, @@ -267,12 +267,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda assert not file1_1.is_file() assert not file2_1.is_file() - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='1m') - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='CFI/BTC', - timeframe='1m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='1m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='CFI/BTC', + timeframe='1m') assert not exchange._pairs_last_refresh_time assert file1_1.is_file() assert file2_1.is_file() @@ -284,12 +284,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda assert not file1_5.is_file() assert not file2_5.is_file() - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='5m') - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='CFI/BTC', - timeframe='5m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='5m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='CFI/BTC', + timeframe='5m') assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() @@ -307,8 +307,8 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) - download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') - download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') + _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') + _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') assert json_dump_mock.call_count == 2 @@ -324,9 +324,9 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, _backup_file(file1_1) _backup_file(file1_5) - assert not download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='1m') + assert not _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='1m') # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) @@ -570,7 +570,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -591,7 +591,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch( @@ -611,7 +611,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -646,8 +646,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert not file1.is_file() - assert download_trades_history(datadir=testdatadir, exchange=exchange, - pair='ETH/BTC') + assert _download_trades_history(datadir=testdatadir, exchange=exchange, + pair='ETH/BTC') assert log_has("New Amount of trades: 5", caplog) assert file1.is_file() @@ -657,8 +657,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) - assert not download_trades_history(datadir=testdatadir, exchange=exchange, - pair='ETH/BTC') + assert not _download_trades_history(datadir=testdatadir, exchange=exchange, + pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) From fa968996ed980df63aa139180af0ec5799b74b26 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 22:01:26 +0300 Subject: [PATCH 023/128] Remove useless check --- freqtrade/data/history.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 15c5ed451..5176a9b19 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -295,11 +295,6 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ - if not exchange: - raise OperationalException( - "Exchange needs to be initialized when downloading pair history data" - ) - try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' From a6fc743d854e33ddd44fb11cea277ab1a6482f86 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 22:12:26 +0300 Subject: [PATCH 024/128] Align code in _download_*_history() --- freqtrade/data/history.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 5176a9b19..79500c512 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -307,11 +307,12 @@ def _download_pair_history(datadir: Path, logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given - new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms if since_ms - else + new_data = exchange.get_historic_ohlcv(pair=pair, + timeframe=timeframe, + since_ms=since_ms if since_ms else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000) + days=-30).float_timestamp) * 1000 + ) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) @@ -376,11 +377,11 @@ def _download_trades_history(datadir: Path, logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None') logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None') + # Default since_ms to 30 days if nothing is given new_trades = exchange.get_historic_trades(pair=pair, since=since if since else int(arrow.utcnow().shift( days=-30).float_timestamp) * 1000, - # until=xxx, from_id=from_id, ) trades.extend(new_trades[1]) From 9cea5cd4427cbabd877860d34bb3a8cad32a2149 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2019 20:38:36 +0100 Subject: [PATCH 025/128] Add documentation about ohlcv_partial_candle --- docs/developer.md | 6 ++++-- docs/exchanges.md | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index fe37c140e..5b07aff03 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -183,17 +183,19 @@ raw = ct.fetch_ohlcv(pair, timeframe=timeframe) # convert to dataframe df1 = parse_ticker_dataframe(raw, timeframe, pair=pair, drop_incomplete=False) -print(df1["date"].tail(1)) +print(df1.tail(1)) print(datetime.utcnow()) ``` ``` output -19 2019-06-08 00:00:00+00:00 + date open high low close volume +499 2019-06-08 00:00:00+00:00 0.000007 0.000007 0.000007 0.000007 26264344.0 2019-06-09 12:30:27.873327 ``` The output will show the last entry from the Exchange as well as the current UTC date. If the day shows the same day, then the last candle can be assumed as incomplete and should be dropped (leave the setting `"ohlcv_partial_candle"` from the exchange-class untouched / True). Otherwise, set `"ohlcv_partial_candle"` to `False` to not drop Candles (shown in the example above). +Another way is to run this command multiple times in a row and observe if the volume is changing (while the date remains the same). ## Updating example notebooks diff --git a/docs/exchanges.md b/docs/exchanges.md index 5bd283a69..d836b4c32 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -61,3 +61,24 @@ print(res) ```shell $ pip3 install web3 ``` + +### Send incomplete candles to the strategy + +Most exchanges return incomplete candles via their ohlcv / klines interface. +By default, Freqtrade assumes that incomplete candles are returned and removes the last candle assuming it's an incomplete candle. + +Wether your exchange returns incomplete candles or not can be checked using [the helper script](developer.md#Incomplete-candles) from the Contributor documentation. + +If the exchange does return incomplete candles and you would like to have incomplete candles in your strategy, you can set the following parameter in the configuration file. + +``` json +{ + + "exchange": { + "_ft_has_params": {"ohlcv_partial_candle": false} + } +} +``` + +!!! Warning "Danger of repainting" + Changing this parameter makes the strategy responsible to avoid repainting and handle this accordingly. Doing this is therefore not recommended, and should only be performed by experienced users who are fully aware of the impact this setting has. From 0277cd82ea66cd019138f0f45caaeed3b0afc187 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 23:25:57 +0300 Subject: [PATCH 026/128] Make mypy happy --- freqtrade/data/history.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 79500c512..2c01f9f48 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -295,6 +295,11 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ + if not exchange: + raise OperationalException( + "Exchange needs to be initialized when downloading pair history data" + ) + try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' From 707c5668a52f1956a458cf2255b26ce89aea1a7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 06:11:44 +0100 Subject: [PATCH 027/128] Fix typo --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index d836b4c32..76fa81f4a 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -67,7 +67,7 @@ $ pip3 install web3 Most exchanges return incomplete candles via their ohlcv / klines interface. By default, Freqtrade assumes that incomplete candles are returned and removes the last candle assuming it's an incomplete candle. -Wether your exchange returns incomplete candles or not can be checked using [the helper script](developer.md#Incomplete-candles) from the Contributor documentation. +Whether your exchange returns incomplete candles or not can be checked using [the helper script](developer.md#Incomplete-candles) from the Contributor documentation. If the exchange does return incomplete candles and you would like to have incomplete candles in your strategy, you can set the following parameter in the configuration file. From 0b5354f13db399a21b06275aeb0d6edc3097ee2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 06:58:10 +0100 Subject: [PATCH 028/128] Add required arguments to Trade method --- tests/strategy/test_interface.py | 3 ++ tests/test_freqtradebot.py | 73 +++++++++++++++++++++++--------- tests/test_persistence.py | 33 ++++++++++++--- 3 files changed, 82 insertions(+), 27 deletions(-) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 5519b1a34..605622b8f 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -125,6 +125,7 @@ def test_min_roi_reached(default_conf, fee) -> None: trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, open_date=arrow.utcnow().shift(hours=-1).datetime, fee_open=fee.return_value, fee_close=fee.return_value, @@ -162,6 +163,7 @@ def test_min_roi_reached2(default_conf, fee) -> None: trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, open_date=arrow.utcnow().shift(hours=-1).datetime, fee_open=fee.return_value, fee_close=fee.return_value, @@ -195,6 +197,7 @@ def test_min_roi_reached3(default_conf, fee) -> None: trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, open_date=arrow.utcnow().shift(hours=-1).datetime, fee_open=fee.return_value, fee_close=fee.return_value, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 18f5a461a..9dff73322 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1512,13 +1512,15 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=limit_buy_order['amount']) - trade = Trade() - # Mock session away - Trade.session = MagicMock() - trade.open_order_id = '123' - trade.open_fee = 0.001 + trade = Trade( + open_order_id=123, + fee_open=0.001, + fee_close=0.001, + open_rate=0.01, + open_date=arrow.utcnow().datetime, + amount=11, + ) # Add datetime explicitly since sqlalchemy defaults apply only once written to database - trade.open_date = arrow.utcnow().datetime freqtrade.update_trade_state(trade) # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) @@ -1541,7 +1543,7 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No assert log_has_re('Found open order for.*', caplog) -def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, mocker): +def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # get_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.get_order', MagicMock(side_effect=ValueError)) @@ -1554,6 +1556,8 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ amount=amount, exchange='binance', open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, open_order_id="123456", is_open=True, ) @@ -1562,7 +1566,7 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ assert trade.amount == limit_buy_order['amount'] -def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_order, +def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_order, fee, limit_buy_order, mocker, caplog): trades_for_order[0]['amount'] = limit_buy_order['amount'] + 1e-14 mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) @@ -1577,6 +1581,8 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_ amount=amount, exchange='binance', open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, open_order_id="123456", is_open=True, open_date=arrow.utcnow().datetime, @@ -2972,7 +2978,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, assert trade.sell_reason == SellType.STOP_LOSS.value -def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker): +def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) patch_RPCManager(mocker) patch_exchange(mocker) @@ -2982,6 +2988,8 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca amount=amount, exchange='binance', open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) @@ -2994,7 +3002,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca caplog) -def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): +def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) patch_RPCManager(mocker) @@ -3005,6 +3013,8 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): amount=amount, exchange='binance', open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) @@ -3017,7 +3027,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): caplog) -def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mocker): +def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): trades_for_order[0]['fee']['currency'] = 'ETH' patch_RPCManager(mocker) @@ -3028,6 +3038,8 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo pair='LTC/ETH', amount=amount, exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.245441, open_order_id="123456" ) @@ -3038,7 +3050,8 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo assert freqtrade.get_real_amount(trade, buy_order_fee) == amount -def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_order_fee, mocker): +def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_order_fee, + fee, mocker): limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': None} @@ -3052,6 +3065,8 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ pair='LTC/ETH', amount=amount, exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.245441, open_order_id="123456" ) @@ -3062,7 +3077,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ assert freqtrade.get_real_amount(trade, limit_buy_order) == amount -def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mocker): +def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, mocker): trades_for_order[0]['fee']['currency'] = 'BNB' trades_for_order[0]['fee']['cost'] = 0.00094518 @@ -3074,6 +3089,8 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock pair='LTC/ETH', amount=amount, exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.245441, open_order_id="123456" ) @@ -3084,7 +3101,7 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock assert freqtrade.get_real_amount(trade, buy_order_fee) == amount -def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, mocker): +def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker): patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) @@ -3093,6 +3110,8 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c pair='LTC/ETH', amount=amount, exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.245441, open_order_id="123456" ) @@ -3106,7 +3125,8 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c caplog) -def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, caplog, mocker): +def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, + caplog, mocker): limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} @@ -3119,6 +3139,8 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee pair='LTC/ETH', amount=amount, exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.245441, open_order_id="123456" ) @@ -3132,7 +3154,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee caplog) -def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, mocker): +def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} @@ -3144,6 +3166,8 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order pair='LTC/ETH', amount=amount, exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.245441, open_order_id="123456" ) @@ -3154,7 +3178,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order assert freqtrade.get_real_amount(trade, limit_buy_order) == amount -def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_fee, mocker): +def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_fee, fee, mocker): limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['amount'] = limit_buy_order['amount'] - 0.001 @@ -3167,6 +3191,8 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ amount=amount, exchange='binance', open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) @@ -3177,7 +3203,7 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ freqtrade.get_real_amount(trade, limit_buy_order) -def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, buy_order_fee, +def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, buy_order_fee, fee, mocker): # Floats should not be compared directly. limit_buy_order = deepcopy(buy_order_fee) @@ -3191,6 +3217,8 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b pair='LTC/ETH', amount=amount, exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.245441, open_order_id="123456" ) @@ -3202,7 +3230,7 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b abs_tol=MATH_CLOSE_PREC,) -def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, mocker): +def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, fee, mocker): # Remove "Currency" from fee dict trades_for_order[0]['fee'] = {'cost': 0.008} @@ -3215,6 +3243,9 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, amount=amount, exchange='binance', open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) @@ -3223,7 +3254,7 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, assert freqtrade.get_real_amount(trade, buy_order_fee) == amount -def test_get_real_amount_open_trade(default_conf, mocker): +def test_get_real_amount_open_trade(default_conf, fee, mocker): patch_RPCManager(mocker) patch_exchange(mocker) amount = 12345 @@ -3232,6 +3263,8 @@ def test_get_real_amount_open_trade(default_conf, mocker): amount=amount, exchange='binance', open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, open_order_id="123456" ) order = { diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 231a1d2e2..67c61172c 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -136,12 +136,13 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): id=2, pair='ETH/BTC', stake_amount=0.001, + open_rate=0.01, + amount=5, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', ) assert trade.open_order_id is None - assert trade.open_rate is None assert trade.close_profit is None assert trade.close_date is None @@ -173,6 +174,8 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): id=1, pair='ETH/BTC', stake_amount=0.001, + amount=5, + open_rate=0.01, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -205,6 +208,8 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + open_rate=0.01, + amount=5, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -212,7 +217,7 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): trade.open_order_id = 'something' trade.update(limit_buy_order) - assert trade.calc_open_trade_price() == 0.0010024999999225068 + assert trade._calc_open_trade_price() == 0.0010024999999225068 trade.update(limit_sell_order) assert trade.calc_close_trade_price() == 0.0010646656050132426 @@ -229,6 +234,8 @@ def test_calc_close_trade_price_exception(limit_buy_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + open_rate=0.1, + amount=5, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -244,13 +251,14 @@ def test_update_open_order(limit_buy_order): trade = Trade( pair='ETH/BTC', stake_amount=1.00, + open_rate=0.01, + amount=5, fee_open=0.1, fee_close=0.1, exchange='bittrex', ) assert trade.open_order_id is None - assert trade.open_rate is None assert trade.close_profit is None assert trade.close_date is None @@ -258,7 +266,6 @@ def test_update_open_order(limit_buy_order): trade.update(limit_buy_order) assert trade.open_order_id is None - assert trade.open_rate is None assert trade.close_profit is None assert trade.close_date is None @@ -268,6 +275,8 @@ def test_update_invalid_order(limit_buy_order): trade = Trade( pair='ETH/BTC', stake_amount=1.00, + amount=5, + open_rate=0.001, fee_open=0.1, fee_close=0.1, exchange='bittrex', @@ -282,6 +291,8 @@ def test_calc_open_trade_price(limit_buy_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, + open_rate=0.00001099, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -290,10 +301,10 @@ def test_calc_open_trade_price(limit_buy_order, fee): trade.update(limit_buy_order) # Buy @ 0.00001099 # Get the open rate price with the standard fee rate - assert trade.calc_open_trade_price() == 0.0010024999999225068 - + assert trade._calc_open_trade_price() == 0.0010024999999225068 + trade.fee_open = 0.003 # Get the open rate price with a custom fee rate - assert trade.calc_open_trade_price(fee=0.003) == 0.001002999999922468 + assert trade._calc_open_trade_price() == 0.001002999999922468 @pytest.mark.usefixtures("init_persistence") @@ -301,6 +312,8 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, + open_rate=0.00001099, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -324,6 +337,8 @@ def test_calc_profit(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, + open_rate=0.00001099, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -356,6 +371,8 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, + open_rate=0.00001099, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -630,6 +647,7 @@ def test_adjust_stop_loss(fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -681,6 +699,7 @@ def test_adjust_min_max_rates(fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, + amount=5, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', From 307ade6251a7ec3a5e0a914204c9cf7da07c6866 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 07:02:02 +0100 Subject: [PATCH 029/128] Cache open_trade_price --- freqtrade/persistence.py | 23 ++++++++++++++++------- tests/test_persistence.py | 2 ++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 735c740c3..e1a3ad713 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -86,7 +86,7 @@ def check_migrate(engine) -> None: logger.debug(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'stop_loss_pct'): + if not has_column(cols, 'open_trade_price'): logger.info(f'Running database migration - backup available as {table_back_name}') fee_open = get_column_def(cols, 'fee_open', 'fee') @@ -104,6 +104,8 @@ def check_migrate(engine) -> None: sell_reason = get_column_def(cols, 'sell_reason', 'null') strategy = get_column_def(cols, 'strategy', 'null') ticker_interval = get_column_def(cols, 'ticker_interval', 'null') + open_trade_price = get_column_def(cols, 'open_trade_price', + f'amount * open_rate * (1 + {fee_open})') # Schema migration necessary engine.execute(f"alter table trades rename to {table_back_name}") @@ -121,7 +123,7 @@ def check_migrate(engine) -> None: stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, stoploss_order_id, stoploss_last_update, max_rate, min_rate, sell_reason, strategy, - ticker_interval + ticker_interval, open_trade_price ) select id, lower(exchange), case @@ -140,7 +142,8 @@ def check_migrate(engine) -> None: {initial_stop_loss_pct} initial_stop_loss_pct, {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, - {strategy} strategy, {ticker_interval} ticker_interval + {strategy} strategy, {ticker_interval} ticker_interval, + {open_trade_price} open_trade_price from {table_back_name} """) @@ -182,6 +185,7 @@ class Trade(_DECL_BASE): fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) open_rate_requested = Column(Float) + open_trade_price = Column(Float) close_rate = Column(Float) close_rate_requested = Column(Float) close_profit = Column(Float) @@ -210,6 +214,10 @@ class Trade(_DECL_BASE): strategy = Column(String, nullable=True) ticker_interval = Column(Integer, nullable=True) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.open_trade_price = self._calc_open_trade_price() + def __repr__(self): open_since = self.open_date.strftime('%Y-%m-%d %H:%M:%S') if self.is_open else 'closed' @@ -302,6 +310,7 @@ class Trade(_DECL_BASE): # Update open rate and actual amount self.open_rate = Decimal(order['price']) self.amount = Decimal(order['amount']) + self.open_trade_price = self._calc_open_trade_price() logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) self.open_order_id = None elif order_type in ('market', 'limit') and order['side'] == 'sell': @@ -331,7 +340,7 @@ class Trade(_DECL_BASE): self ) - def calc_open_trade_price(self, fee: Optional[float] = None) -> float: + def _calc_open_trade_price(self) -> float: """ Calculate the open_rate including fee. :param fee: fee to use on the open rate (optional). @@ -339,7 +348,7 @@ class Trade(_DECL_BASE): :return: Price in of the open trade incl. Fees """ buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) - fees = buy_trade * Decimal(fee or self.fee_open) + fees = buy_trade * Decimal(self.fee_open) return float(buy_trade + fees) def calc_close_trade_price(self, rate: Optional[float] = None, @@ -369,7 +378,7 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: profit in stake currency as float """ - open_trade_price = self.calc_open_trade_price() + open_trade_price = self._calc_open_trade_price() close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), fee=(fee or self.fee_close) @@ -386,7 +395,7 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit in percentage as float """ - open_trade_price = self.calc_open_trade_price() + open_trade_price = self._calc_open_trade_price() close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), fee=(fee or self.fee_close) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 67c61172c..2cb79de98 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -498,6 +498,7 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 + assert trade.open_trade_price == 0.26758131848350003 def test_migrate_new(mocker, default_conf, fee, caplog): @@ -580,6 +581,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak2", caplog) assert log_has("Running database migration - backup available as trades_bak2", caplog) + assert trade.open_trade_price == 0.26758131848350003 def test_migrate_mid_state(mocker, default_conf, fee, caplog): From 861a7834fc32556b7d59be02c872c66f7a49efb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 07:08:36 +0100 Subject: [PATCH 030/128] Call calc_open_price() whenever necessary --- freqtrade/freqtradebot.py | 2 ++ freqtrade/persistence.py | 17 +++++++++++------ tests/test_persistence.py | 4 ++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5c3ef64b1..a7ca67dcf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -555,6 +555,7 @@ class FreqtradeBot: order['amount'] = new_amount # Fee was applied, so set to 0 trade.fee_open = 0 + trade.recalc_open_trade_price() except DependencyException as exception: logger.warning("Could not update trade amount: %s", exception) @@ -850,6 +851,7 @@ class FreqtradeBot: trade.amount = new_amount # Fee was applied, so set to 0 trade.fee_open = 0 + trade.recalc_open_trade_price() except DependencyException as e: logger.warning("Could not update trade amount: %s", e) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index e1a3ad713..354f713bc 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -216,7 +216,7 @@ class Trade(_DECL_BASE): def __init__(self, **kwargs): super().__init__(**kwargs) - self.open_trade_price = self._calc_open_trade_price() + self.recalc_open_trade_price() def __repr__(self): open_since = self.open_date.strftime('%Y-%m-%d %H:%M:%S') if self.is_open else 'closed' @@ -310,7 +310,7 @@ class Trade(_DECL_BASE): # Update open rate and actual amount self.open_rate = Decimal(order['price']) self.amount = Decimal(order['amount']) - self.open_trade_price = self._calc_open_trade_price() + self.recalc_open_trade_price() logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) self.open_order_id = None elif order_type in ('market', 'limit') and order['side'] == 'sell': @@ -351,6 +351,13 @@ class Trade(_DECL_BASE): fees = buy_trade * Decimal(self.fee_open) return float(buy_trade + fees) + def recalc_open_trade_price(self) -> None: + """ + Recalculate open_trade_price. + Must be called whenever open_rate or fee_open is changed. + """ + self.open_trade_price - self._calc_open_trade_price() + def calc_close_trade_price(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ @@ -378,12 +385,11 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: profit in stake currency as float """ - open_trade_price = self._calc_open_trade_price() close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit = close_trade_price - open_trade_price + profit = close_trade_price - self.open_trade_price return float(f"{profit:.8f}") def calc_profit_percent(self, rate: Optional[float] = None, @@ -395,12 +401,11 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit in percentage as float """ - open_trade_price = self._calc_open_trade_price() close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit_percent = (close_trade_price / open_trade_price) - 1 + profit_percent = (close_trade_price / self.open_trade_price) - 1 return float(f"{profit_percent:.8f}") @staticmethod diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 2cb79de98..06df73cf9 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -498,7 +498,7 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_price == 0.26758131848350003 + assert trade.open_trade_price == trade._calc_open_trade_price() def test_migrate_new(mocker, default_conf, fee, caplog): @@ -581,7 +581,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak2", caplog) assert log_has("Running database migration - backup available as trades_bak2", caplog) - assert trade.open_trade_price == 0.26758131848350003 + assert trade.open_trade_price == trade._calc_open_trade_price() def test_migrate_mid_state(mocker, default_conf, fee, caplog): From 362a40db6f89be6bd0fafb77f5f51b6f55345c45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 07:09:56 +0100 Subject: [PATCH 031/128] Update docstring --- freqtrade/persistence.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 354f713bc..21e1b82ce 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -342,9 +342,7 @@ class Trade(_DECL_BASE): def _calc_open_trade_price(self) -> float: """ - Calculate the open_rate including fee. - :param fee: fee to use on the open rate (optional). - If rate is not set self.fee will be used + Calculate the open_rate including open_fee. :return: Price in of the open trade incl. Fees """ buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) From cbd10309f514862c7bb87eab81be845b7214c134 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 07:13:08 +0100 Subject: [PATCH 032/128] Add mid-state test --- tests/test_persistence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 06df73cf9..0bde2f673 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -641,6 +641,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 + assert trade.open_trade_price == trade._calc_open_trade_price() assert log_has("trying trades_bak0", caplog) assert log_has("Running database migration - backup available as trades_bak0", caplog) From 539b5627fdd67f0e245cce67a9b4428f94d8253f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 08:31:44 +0100 Subject: [PATCH 033/128] Fix typo --- freqtrade/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 21e1b82ce..61dee0414 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -354,7 +354,7 @@ class Trade(_DECL_BASE): Recalculate open_trade_price. Must be called whenever open_rate or fee_open is changed. """ - self.open_trade_price - self._calc_open_trade_price() + self.open_trade_price = self._calc_open_trade_price() def calc_close_trade_price(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: From a2964afd42c4ccf84ff315e140e0064efd62d6a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 08:53:30 +0100 Subject: [PATCH 034/128] Rename profit_percent to profit_ratio to be consistent --- freqtrade/data/btanalysis.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/persistence.py | 21 +++++++++++---------- freqtrade/rpc/rpc.py | 8 ++++---- freqtrade/strategy/interface.py | 6 +++--- tests/rpc/test_rpc_apiserver.py | 4 ++-- tests/test_freqtradebot.py | 3 ++- tests/test_persistence.py | 12 ++++++------ 9 files changed, 32 insertions(+), 30 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 379c80060..2fc931a9b 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -108,7 +108,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: trades = pd.DataFrame([(t.pair, t.open_date.replace(tzinfo=timezone.utc), t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None, - t.calc_profit(), t.calc_profit_percent(), + t.calc_profit(), t.calc_profit_ratio(), t.open_rate, t.close_rate, t.amount, (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2) if t.close_date else None), diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a7ca67dcf..8ae027fa2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -950,7 +950,7 @@ class FreqtradeBot: profit_trade = trade.calc_profit(rate=profit_rate) # Use cached ticker here - it was updated seconds ago. current_rate = self.get_sell_rate(trade.pair, False) - profit_percent = trade.calc_profit_percent(profit_rate) + profit_percent = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_percent > 0 else "loss" msg = { diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 064a2f6ba..fc60bd310 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -329,7 +329,7 @@ class Backtesting: closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) return BacktestResult(pair=pair, - profit_percent=trade.calc_profit_percent(rate=closerate), + profit_percent=trade.calc_profit_ratio(rate=closerate), profit_abs=trade.calc_profit(rate=closerate), open_time=buy_row.date, close_time=sell_row.date, @@ -345,7 +345,7 @@ class Backtesting: # no sell condition found - trade stil open at end of backtest period sell_row = partial_ticker[-1] bt_res = BacktestResult(pair=pair, - profit_percent=trade.calc_profit_percent(rate=sell_row.open), + profit_percent=trade.calc_profit_ratio(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 61dee0414..10896baaa 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -185,6 +185,7 @@ class Trade(_DECL_BASE): fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) open_rate_requested = Column(Float) + # open_trade_price - calcuated via _calc_open_trade_price open_trade_price = Column(Float) close_rate = Column(Float) close_rate_requested = Column(Float) @@ -331,7 +332,7 @@ class Trade(_DECL_BASE): and marks trade as closed """ self.close_rate = Decimal(rate) - self.close_profit = self.calc_profit_percent() + self.close_profit = self.calc_profit_ratio() self.close_date = datetime.utcnow() self.is_open = False self.open_order_id = None @@ -361,9 +362,9 @@ class Trade(_DECL_BASE): """ Calculate the close_rate including fee :param fee: fee to use on the close rate (optional). - If rate is not set self.fee will be used + If rate is not set self.fee will be used :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used + If rate is not set self.close_rate will be used :return: Price in BTC of the open trade """ if rate is None and not self.close_rate: @@ -378,9 +379,9 @@ class Trade(_DECL_BASE): """ Calculate the absolute profit in stake currency between Close and Open trade :param fee: fee to use on the close rate (optional). - If rate is not set self.fee will be used + If rate is not set self.fee will be used :param rate: close rate to compare with (optional). - If rate is not set self.close_rate will be used + If rate is not set self.close_rate will be used :return: profit in stake currency as float """ close_trade_price = self.calc_close_trade_price( @@ -390,14 +391,14 @@ class Trade(_DECL_BASE): profit = close_trade_price - self.open_trade_price return float(f"{profit:.8f}") - def calc_profit_percent(self, rate: Optional[float] = None, - fee: Optional[float] = None) -> float: + def calc_profit_ratio(self, rate: Optional[float] = None, + fee: Optional[float] = None) -> float: """ - Calculates the profit in percentage (including fee). + Calculates the profit as ratio (including fee). :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used + If rate is not set self.close_rate will be used :param fee: fee to use on the close rate (optional). - :return: profit in percentage as float + :return: profit ratio as float """ close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 84b72fe18..3b4b7570a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -123,7 +123,7 @@ class RPC: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) except DependencyException: current_rate = NAN - current_profit = trade.calc_profit_percent(current_rate) + current_profit = trade.calc_profit_ratio(current_rate) fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' if trade.close_profit else None) trade_dict = trade.to_json() @@ -151,7 +151,7 @@ class RPC: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) except DependencyException: current_rate = NAN - trade_perc = (100 * trade.calc_profit_percent(current_rate)) + trade_perc = (100 * trade.calc_profit_ratio(current_rate)) trade_profit = trade.calc_profit(current_rate) profit_str = f'{trade_perc:.2f}%' if self._fiat_converter: @@ -240,7 +240,7 @@ class RPC: durations.append((trade.close_date - trade.open_date).total_seconds()) if not trade.is_open: - profit_percent = trade.calc_profit_percent() + profit_percent = trade.calc_profit_ratio() profit_closed_coin.append(trade.calc_profit()) profit_closed_perc.append(profit_percent) else: @@ -249,7 +249,7 @@ class RPC: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) except DependencyException: current_rate = NAN - profit_percent = trade.calc_profit_percent(rate=current_rate) + profit_percent = trade.calc_profit_ratio(rate=current_rate) profit_all_coin.append( trade.calc_profit(rate=trade.close_rate or current_rate) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 2b3a6194f..985ff37de 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -296,7 +296,7 @@ class IStrategy(ABC): """ # Set current rate to low for backtesting sell current_rate = low or rate - current_profit = trade.calc_profit_percent(current_rate) + current_profit = trade.calc_profit_ratio(current_rate) trade.adjust_min_max_rates(high or current_rate) @@ -311,7 +311,7 @@ class IStrategy(ABC): # Set current rate to high for backtesting sell current_rate = high or rate - current_profit = trade.calc_profit_percent(current_rate) + current_profit = trade.calc_profit_ratio(current_rate) config_ask_strategy = self.config.get('ask_strategy', {}) if buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False): @@ -360,7 +360,7 @@ class IStrategy(ABC): sl_offset = self.trailing_stop_positive_offset # Make sure current_profit is calculated using high for backtesting. - high_profit = current_profit if not high else trade.calc_profit_percent(high) + high_profit = current_profit if not high else trade.calc_profit_ratio(high) # Don't update stoploss if trailing_only_offset_is_reached is true. if not (self.trailing_only_offset_is_reached and high_profit < sl_offset): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index ebb70bdf8..f1e3421c5 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -381,7 +381,7 @@ def test_api_performance(botclient, mocker, ticker, fee): close_rate=0.265441, ) - trade.close_profit = trade.calc_profit_percent() + trade.close_profit = trade.calc_profit_ratio() Trade.session.add(trade) trade = Trade( @@ -396,7 +396,7 @@ def test_api_performance(botclient, mocker, ticker, fee): fee_open=fee.return_value, close_rate=0.391 ) - trade.close_profit = trade.calc_profit_percent() + trade.close_profit = trade.calc_profit_ratio() Trade.session.add(trade) Trade.session.flush() diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 9dff73322..341bc021f 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1543,7 +1543,8 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No assert log_has_re('Found open order for.*', caplog) -def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, mocker): +def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, + mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # get_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.get_order', MagicMock(side_effect=ValueError)) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 0bde2f673..25ad8b6a7 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -226,7 +226,7 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): assert trade.calc_profit() == 0.00006217 # Profit in percent - assert trade.calc_profit_percent() == 0.06201058 + assert trade.calc_profit_ratio() == 0.06201058 @pytest.mark.usefixtures("init_persistence") @@ -367,7 +367,7 @@ def test_calc_profit(limit_buy_order, limit_sell_order, fee): @pytest.mark.usefixtures("init_persistence") -def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee): +def test_calc_profit_ratio(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, @@ -381,17 +381,17 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee): trade.update(limit_buy_order) # Buy @ 0.00001099 # Get percent of profit with a custom rate (Higher than open rate) - assert trade.calc_profit_percent(rate=0.00001234) == 0.11723875 + assert trade.calc_profit_ratio(rate=0.00001234) == 0.11723875 # Get percent of profit with a custom rate (Lower than open rate) - assert trade.calc_profit_percent(rate=0.00000123) == -0.88863828 + assert trade.calc_profit_ratio(rate=0.00000123) == -0.88863828 # Test when we apply a Sell order. Sell higher than open rate @ 0.00001173 trade.update(limit_sell_order) - assert trade.calc_profit_percent() == 0.06201058 + assert trade.calc_profit_ratio() == 0.06201058 # Test with a custom fee rate on the close trade - assert trade.calc_profit_percent(fee=0.003) == 0.06147824 + assert trade.calc_profit_ratio(fee=0.003) == 0.06147824 @pytest.mark.usefixtures("init_persistence") From 86de88ed480bfd16ec8c875b79c3d4298b0be849 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 09:36:26 +0100 Subject: [PATCH 035/128] Align usage of history import in test --- tests/data/test_history.py | 100 +++++++++++++++---------------------- 1 file changed, 40 insertions(+), 60 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index c304aa97d..708702ab2 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -12,16 +12,17 @@ from pandas import DataFrame from freqtrade import OperationalException from freqtrade.configuration import TimeRange -from freqtrade.data import history -from freqtrade.data.history import (_load_cached_data_for_updating, - _download_pair_history, +from freqtrade.data.history import (_download_pair_history, _download_trades_history, - convert_trades_to_ohlcv, + _load_cached_data_for_updating, + convert_trades_to_ohlcv, get_timeframe, + load_data, load_pair_history, load_tickerdata_file, pair_data_filename, pair_trades_filename, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, - trim_tickerlist) + trim_dataframe, trim_tickerlist, + validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy @@ -64,7 +65,7 @@ def _clean_test_file(file: Path) -> None: def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> None: - ld = history.load_pair_history(pair='UNITTEST/BTC', timeframe='30m', datadir=testdatadir) + ld = load_pair_history(pair='UNITTEST/BTC', timeframe='30m', datadir=testdatadir) assert isinstance(ld, DataFrame) assert not log_has( 'Download history data for pair: "UNITTEST/BTC", timeframe: 30m ' @@ -73,7 +74,7 @@ def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> No def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> None: - ld = history.load_pair_history(pair='UNITTEST/BTC', timeframe='7m', datadir=testdatadir) + ld = load_pair_history(pair='UNITTEST/BTC', timeframe='7m', datadir=testdatadir) assert isinstance(ld, DataFrame) assert ld.empty assert log_has( @@ -86,7 +87,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history) file = testdatadir / 'UNITTEST_BTC-1m.json' _backup_file(file, copy_file=True) - history.load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC']) + load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC']) assert file.is_file() assert not log_has( 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' @@ -99,10 +100,9 @@ def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file', MagicMock(return_value=None)) timerange = TimeRange('date', None, 1510639620, 0) - history.load_pair_history(pair='UNITTEST/BTC', timeframe='1m', - datadir=testdatadir, timerange=timerange, - startup_candles=20, - ) + load_pair_history(pair='UNITTEST/BTC', timeframe='1m', + datadir=testdatadir, timerange=timerange, + startup_candles=20,) assert ltfmock.call_count == 1 assert ltfmock.call_args_list[0][1]['timerange'] != timerange @@ -121,9 +121,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, _backup_file(file) # do not download a new pair if refresh_pairs isn't set - history.load_pair_history(datadir=testdatadir, - timeframe='1m', - pair='MEME/BTC') + load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') assert not file.is_file() assert log_has( 'No history data for pair: "MEME/BTC", timeframe: 1m. ' @@ -131,22 +129,16 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, ) # download a new pair if refresh_pairs is set - history.load_pair_history(datadir=testdatadir, - timeframe='1m', - refresh_pairs=True, - exchange=exchange, - pair='MEME/BTC') + load_pair_history(datadir=testdatadir, timeframe='1m', + refresh_pairs=True, exchange=exchange, pair='MEME/BTC') assert file.is_file() assert log_has_re( 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): - history.load_pair_history(datadir=testdatadir, - timeframe='1m', - refresh_pairs=True, - exchange=None, - pair='MEME/BTC') + load_pair_history(datadir=testdatadir, timeframe='1m', + refresh_pairs=True, exchange=None, pair='MEME/BTC') _clean_test_file(file) @@ -351,10 +343,8 @@ def test_load_partial_missing(testdatadir, caplog) -> None: # Make sure we start fresh - test missing data at start start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') - tickerdata = history.load_data(testdatadir, '5m', ['UNITTEST/BTC'], - startup_candles=20, - timerange=TimeRange('date', 'date', - start.timestamp, end.timestamp)) + tickerdata = load_data(testdatadir, '5m', ['UNITTEST/BTC'], startup_candles=20, + timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) assert log_has( 'Using indicator startup period: 20 ...', caplog ) @@ -369,10 +359,8 @@ def test_load_partial_missing(testdatadir, caplog) -> None: caplog.clear() start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') - tickerdata = history.load_data(datadir=testdatadir, timeframe='5m', - pairs=['UNITTEST/BTC'], - timerange=TimeRange('date', 'date', - start.timestamp, end.timestamp)) + tickerdata = load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], + timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(tickerdata['UNITTEST/BTC']) @@ -385,7 +373,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: def test_init(default_conf, mocker) -> None: exchange = get_patched_exchange(mocker, default_conf) - assert {} == history.load_data( + assert {} == load_data( datadir='', exchange=exchange, pairs=[], @@ -447,7 +435,7 @@ def test_trim_tickerlist(testdatadir) -> None: def test_trim_dataframe(testdatadir) -> None: - data = history.load_data( + data = load_data( datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'] @@ -458,7 +446,7 @@ def test_trim_dataframe(testdatadir) -> None: # Remove first 30 minutes (1800 s) tr = TimeRange('date', None, min_date + 1800, 0) - data_modify = history.trim_dataframe(data_modify, tr) + data_modify = trim_dataframe(data_modify, tr) assert not data_modify.equals(data) assert len(data_modify) < len(data) assert len(data_modify) == len(data) - 30 @@ -468,7 +456,7 @@ def test_trim_dataframe(testdatadir) -> None: data_modify = data.copy() # Remove last 30 minutes (1800 s) tr = TimeRange(None, 'date', 0, max_date - 1800) - data_modify = history.trim_dataframe(data_modify, tr) + data_modify = trim_dataframe(data_modify, tr) assert not data_modify.equals(data) assert len(data_modify) < len(data) assert len(data_modify) == len(data) - 30 @@ -478,7 +466,7 @@ def test_trim_dataframe(testdatadir) -> None: data_modify = data.copy() # Remove first 25 and last 30 minutes (1800 s) tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800) - data_modify = history.trim_dataframe(data_modify, tr) + data_modify = trim_dataframe(data_modify, tr) assert not data_modify.equals(data) assert len(data_modify) < len(data) assert len(data_modify) == len(data) - 55 @@ -515,13 +503,13 @@ def test_get_timeframe(default_conf, mocker, testdatadir) -> None: strategy = DefaultStrategy(default_conf) data = strategy.tickerdata_to_dataframe( - history.load_data( + load_data( datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'] ) ) - min_date, max_date = history.get_timeframe(data) + min_date, max_date = get_timeframe(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' @@ -531,17 +519,17 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) strategy = DefaultStrategy(default_conf) data = strategy.tickerdata_to_dataframe( - history.load_data( + load_data( datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'], fill_up_missing=False ) ) - min_date, max_date = history.get_timeframe(data) + min_date, max_date = get_timeframe(data) caplog.clear() - assert history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', - min_date, max_date, timeframe_to_minutes('1m')) + assert validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', + min_date, max_date, timeframe_to_minutes('1m')) assert len(caplog.record_tuples) == 1 assert log_has( "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values", @@ -554,7 +542,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No timerange = TimeRange('index', 'index', 200, 250) data = strategy.tickerdata_to_dataframe( - history.load_data( + load_data( datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], @@ -562,10 +550,10 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No ) ) - min_date, max_date = history.get_timeframe(data) + min_date, max_date = get_timeframe(data) caplog.clear() - assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', - min_date, max_date, timeframe_to_minutes('5m')) + assert not validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', + min_date, max_date, timeframe_to_minutes('5m')) assert len(caplog.record_tuples) == 0 @@ -668,12 +656,8 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): file1 = testdatadir / 'XRP_ETH-1m.json' file5 = testdatadir / 'XRP_ETH-5m.json' # Compare downloaded dataset with converted dataset - dfbak_1m = history.load_pair_history(datadir=testdatadir, - timeframe="1m", - pair=pair) - dfbak_5m = history.load_pair_history(datadir=testdatadir, - timeframe="5m", - pair=pair) + dfbak_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) + dfbak_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) _backup_file(file1, copy_file=True) _backup_file(file5) @@ -685,12 +669,8 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) # Load new data - df_1m = history.load_pair_history(datadir=testdatadir, - timeframe="1m", - pair=pair) - df_5m = history.load_pair_history(datadir=testdatadir, - timeframe="5m", - pair=pair) + df_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) + df_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) assert df_1m.equals(dfbak_1m) assert df_5m.equals(dfbak_5m) From e1c0c6af7dcf94fe7dc6a982059b6f1a6ccbd2de Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 10:51:49 +0100 Subject: [PATCH 036/128] fix random-seed to failing one --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6a111944..40a6fd075 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: COVERALLS_SERVICE_NAME: travis-ci TRAVIS: "true" run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc + pytest --random-order --cov=freqtrade --cov-config=.coveragerc --random-order-seed=834267 # Allow failure for coveralls # Fake travis environment to get coveralls working correctly export TRAVIS_PULL_REQUEST="https://github.com/${GITHUB_REPOSITORY}/pull/$(cat $GITHUB_EVENT_PATH | jq -r .number)" From 2e2f084f662e02eebc6cb083deb1dbeac145e840 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 11:07:59 +0100 Subject: [PATCH 037/128] Try to clear caplog ... --- .github/workflows/ci.yml | 2 +- tests/test_utils.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40a6fd075..f6a111944 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: COVERALLS_SERVICE_NAME: travis-ci TRAVIS: "true" run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc --random-order-seed=834267 + pytest --random-order --cov=freqtrade --cov-config=.coveragerc # Allow failure for coveralls # Fake travis environment to get coveralls working correctly export TRAVIS_PULL_REQUEST="https://github.com/${GITHUB_REPOSITORY}/pull/$(cat $GITHUB_EVENT_PATH | jq -r .number)" diff --git a/tests/test_utils.py b/tests/test_utils.py index feba1ed59..40ca9ac02 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -444,6 +444,9 @@ def test_create_datadir_failed(caplog): def test_create_datadir(caplog, mocker): + # Ensure that caplog is empty before starting ... + # Should prevent random failures. + caplog.clear() cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock()) csf = mocker.patch("freqtrade.utils.copy_sample_files", MagicMock()) args = [ From 8513a5e2d68721e896e4d3337f8be162847d8c86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 11:35:39 +0100 Subject: [PATCH 038/128] Fix failures in test_main --- .github/workflows/ci.yml | 2 +- tests/test_main.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6a111944..628f169c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,7 +139,7 @@ jobs: - name: Tests run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc + pytest --random-order --cov=freqtrade --cov-config=.coveragerc --random-order-seed=781055 - name: Backtesting run: | diff --git a/tests/test_main.py b/tests/test_main.py index 4e97c375d..03e6a7ce9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -79,6 +79,7 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.worker.Worker._worker', MagicMock(side_effect=KeyboardInterrupt)) patched_configuration_load_config_file(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) args = ['trade', '-c', 'config.json.example'] @@ -98,6 +99,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: MagicMock(side_effect=OperationalException('Oh snap!')) ) patched_configuration_load_config_file(mocker, default_conf) + mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) @@ -120,6 +122,7 @@ def test_main_reload_conf(mocker, default_conf, caplog) -> None: OperationalException("Oh snap!")]) mocker.patch('freqtrade.worker.Worker._worker', worker_mock) patched_configuration_load_config_file(mocker, default_conf) + mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) reconfigure_mock = mocker.patch('freqtrade.worker.Worker._reconfigure', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -143,6 +146,7 @@ def test_reconfigure(mocker, default_conf) -> None: 'freqtrade.worker.Worker._worker', MagicMock(side_effect=OperationalException('Oh snap!')) ) + mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) patched_configuration_load_config_file(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) From 60f89c8c019f6806b28b044e642f8cdab8c60cc9 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 17 Dec 2019 13:43:42 +0300 Subject: [PATCH 039/128] Split refresh from load_data/load_pair_history --- freqtrade/data/history.py | 86 ++++++++++++++++++++---------- freqtrade/edge/__init__.py | 12 ++++- tests/data/test_history.py | 28 +++++++--- tests/edge/test_edge.py | 7 ++- tests/optimize/test_backtesting.py | 4 +- 5 files changed, 97 insertions(+), 40 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 2c01f9f48..62195a005 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -128,39 +128,26 @@ def load_pair_history(pair: str, timeframe: str, datadir: Path, timerange: Optional[TimeRange] = None, - refresh_pairs: bool = False, - exchange: Optional[Exchange] = None, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, ) -> DataFrame: """ - Loads cached ticker history for the given pair. + Load cached ticker history for the given pair. + :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param datadir: Path to the data storage location. :param timerange: Limit data to be loaded to this timerange - :param refresh_pairs: Refresh pairs from exchange. - (Note: Requires exchange to be passed as well.) - :param exchange: Exchange object (needed when using "refresh_pairs") :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :return: DataFrame with ohlcv data, or empty DataFrame """ - timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) - # The user forced the refresh of pairs - if refresh_pairs: - _download_pair_history(datadir=datadir, - exchange=exchange, - pair=pair, - timeframe=timeframe, - timerange=timerange) - pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) if pairdata: @@ -177,33 +164,53 @@ def load_pair_history(pair: str, return DataFrame() +def refresh_pair_history(pair: str, + timeframe: str, + datadir: Path, + exchange: Exchange, + timerange: Optional[TimeRange] = None, + startup_candles: int = 0, + ) -> None: + """ + Refresh cached ticker history for the given pair. + + :param pair: Pair to load data for + :param timeframe: Ticker timeframe (e.g. "5m") + :param datadir: Path to the data storage location. + :param timerange: Limit data to be loaded to this timerange + :param exchange: Exchange object + :param startup_candles: Additional candles to load at the start of the period + """ + timerange_startup = deepcopy(timerange) + if startup_candles > 0 and timerange_startup: + timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) + + _download_pair_history(datadir=datadir, + exchange=exchange, + pair=pair, + timeframe=timeframe, + timerange=timerange) + + def load_data(datadir: Path, timeframe: str, pairs: List[str], - refresh_pairs: bool = False, - exchange: Optional[Exchange] = None, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, startup_candles: int = 0, fail_without_data: bool = False ) -> Dict[str, DataFrame]: """ - Loads ticker history data for a list of pairs + Load ticker history data for a list of pairs. + :param datadir: Path to the data storage location. :param timeframe: Ticker Timeframe (e.g. "5m") :param pairs: List of pairs to load - :param refresh_pairs: Refresh pairs from exchange. - (Note: Requires exchange to be passed as well.) - :param exchange: Exchange object (needed when using "refresh_pairs") :param timerange: Limit data to be loaded to this timerange :param fill_up_missing: Fill missing values with "No action"-candles :param startup_candles: Additional candles to load at the start of the period :param fail_without_data: Raise OperationalException if no data is found. :return: dict(:) - TODO: refresh_pairs is still used by edge to keep the data uptodate. - This should be replaced in the future. Instead, writing the current candles to disk - from dataprovider should be implemented, as this would avoid loading ohlcv data twice. - exchange and refresh_pairs are then not needed here nor in load_pair_history. """ result: Dict[str, DataFrame] = {} if startup_candles > 0 and timerange: @@ -212,8 +219,6 @@ def load_data(datadir: Path, for pair in pairs: hist = load_pair_history(pair=pair, timeframe=timeframe, datadir=datadir, timerange=timerange, - refresh_pairs=refresh_pairs, - exchange=exchange, fill_up_missing=fill_up_missing, startup_candles=startup_candles) if not hist.empty: @@ -224,6 +229,33 @@ def load_data(datadir: Path, return result +def refresh_data(datadir: Path, + timeframe: str, + pairs: List[str], + exchange: Exchange, + timerange: Optional[TimeRange] = None, + startup_candles: int = 0, + ) -> None: + """ + Refresh ticker history data for a list of pairs. + + :param datadir: Path to the data storage location. + :param timeframe: Ticker Timeframe (e.g. "5m") + :param pairs: List of pairs to load + :param exchange: Exchange object + :param timerange: Limit data to be loaded to this timerange + :param startup_candles: Additional candles to load at the start of the period + """ + if startup_candles > 0 and timerange: + logger.info(f'Using indicator startup period: {startup_candles} ...') + + for pair in pairs: + refresh_pair_history(pair=pair, timeframe=timeframe, + datadir=datadir, timerange=timerange, + exchange=exchange, + startup_candles=startup_candles) + + def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: pair_s = pair.replace("/", "_") filename = datadir.joinpath(f'{pair_s}-{timeframe}.json') diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4bc3023a4..b52b2ffdc 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -94,12 +94,20 @@ class Edge: logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using local backtesting data (using whitelist in given config) ...') + if self._refresh_pairs: + history.refresh_data( + datadir=Path(self.config['datadir']), + pairs=pairs, + exchange=self.exchange, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + startup_candles=self.strategy.startup_candle_count, + ) + data = history.load_data( datadir=Path(self.config['datadir']), pairs=pairs, timeframe=self.strategy.ticker_interval, - refresh_pairs=self._refresh_pairs, - exchange=self.exchange, timerange=self._timerange, startup_candles=self.strategy.startup_candle_count, ) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 708702ab2..1b7c2c4a4 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -21,6 +21,7 @@ from freqtrade.data.history import (_download_pair_history, pair_trades_filename, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, + refresh_data, refresh_pair_history, trim_dataframe, trim_tickerlist, validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes @@ -129,16 +130,17 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, ) # download a new pair if refresh_pairs is set - load_pair_history(datadir=testdatadir, timeframe='1m', - refresh_pairs=True, exchange=exchange, pair='MEME/BTC') + refresh_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC', + exchange=exchange) + load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): - load_pair_history(datadir=testdatadir, timeframe='1m', - refresh_pairs=True, exchange=None, pair='MEME/BTC') + refresh_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC', + exchange=None) _clean_test_file(file) @@ -372,12 +374,24 @@ def test_load_partial_missing(testdatadir, caplog) -> None: def test_init(default_conf, mocker) -> None: - exchange = get_patched_exchange(mocker, default_conf) assert {} == load_data( datadir='', - exchange=exchange, pairs=[], - refresh_pairs=True, + timeframe=default_conf['ticker_interval'] + ) + + +def test_init_with_refresh(default_conf, mocker) -> None: + exchange = get_patched_exchange(mocker, default_conf) + refresh_data( + datadir='', + pairs=[], + timeframe=default_conf['ticker_interval'], + exchange=exchange + ) + assert {} == load_data( + datadir='', + pairs=[], timeframe=default_conf['ticker_interval'] ) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index bdb986d6d..3a866c0a8 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -255,8 +255,8 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): assert edge.calculate() is False -def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False, - timerange=None, exchange=None, *args, **kwargs): +def mocked_load_data(datadir, pairs=[], timeframe='0m', + timerange=None, *args, **kwargs): hz = 0.1 base = 0.001 @@ -290,6 +290,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False, def test_edge_process_downloaded_data(mocker, edge_conf): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -301,6 +302,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): def test_edge_process_no_data(mocker, edge_conf, caplog): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -313,6 +315,7 @@ def test_edge_process_no_data(mocker, edge_conf, caplog): def test_edge_process_no_trades(mocker, edge_conf, caplog): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) # Return empty mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[])) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 5086891a6..38a95be7a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -116,8 +116,8 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: assert len(results) == num_results -def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False, - timerange=None, exchange=None, live=False, *args, **kwargs): +def mocked_load_data(datadir, pairs=[], timeframe='0m', + timerange=None, *args, **kwargs): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC", fill_missing=True)} From bbb05b52862dbd04d169a6df010683207545b92b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 11:51:50 +0100 Subject: [PATCH 040/128] Remove fixed random order --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 628f169c2..f6a111944 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,7 +139,7 @@ jobs: - name: Tests run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc --random-order-seed=781055 + pytest --random-order --cov=freqtrade --cov-config=.coveragerc - name: Backtesting run: | From b2796f99b6c9be25af7a86cf67a2740641879cdd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 17 Dec 2019 14:06:21 +0300 Subject: [PATCH 041/128] Remove redundant refresh_pair_history --- freqtrade/data/history.py | 35 +++-------------------------------- tests/data/test_history.py | 10 +++++----- 2 files changed, 8 insertions(+), 37 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 62195a005..3b16e41a9 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -164,34 +164,6 @@ def load_pair_history(pair: str, return DataFrame() -def refresh_pair_history(pair: str, - timeframe: str, - datadir: Path, - exchange: Exchange, - timerange: Optional[TimeRange] = None, - startup_candles: int = 0, - ) -> None: - """ - Refresh cached ticker history for the given pair. - - :param pair: Pair to load data for - :param timeframe: Ticker timeframe (e.g. "5m") - :param datadir: Path to the data storage location. - :param timerange: Limit data to be loaded to this timerange - :param exchange: Exchange object - :param startup_candles: Additional candles to load at the start of the period - """ - timerange_startup = deepcopy(timerange) - if startup_candles > 0 and timerange_startup: - timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) - - _download_pair_history(datadir=datadir, - exchange=exchange, - pair=pair, - timeframe=timeframe, - timerange=timerange) - - def load_data(datadir: Path, timeframe: str, pairs: List[str], @@ -250,10 +222,9 @@ def refresh_data(datadir: Path, logger.info(f'Using indicator startup period: {startup_candles} ...') for pair in pairs: - refresh_pair_history(pair=pair, timeframe=timeframe, - datadir=datadir, timerange=timerange, - exchange=exchange, - startup_candles=startup_candles) + _download_pair_history(pair=pair, timeframe=timeframe, + datadir=datadir, timerange=timerange, + exchange=exchange) def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 1b7c2c4a4..c9b198b39 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -21,7 +21,7 @@ from freqtrade.data.history import (_download_pair_history, pair_trades_filename, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, - refresh_data, refresh_pair_history, + refresh_data, trim_dataframe, trim_tickerlist, validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes @@ -130,8 +130,8 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, ) # download a new pair if refresh_pairs is set - refresh_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC', - exchange=exchange) + refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], + exchange=exchange) load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( @@ -139,8 +139,8 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, 'and store in .*', caplog ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): - refresh_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC', - exchange=None) + refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], + exchange=None) _clean_test_file(file) From 153738961773fbe307aa557fa2ea257c9fe6ad10 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 17 Dec 2019 18:23:31 +0300 Subject: [PATCH 042/128] Remove startup_candles argument in refresh_data --- freqtrade/data/history.py | 5 ----- freqtrade/edge/__init__.py | 1 - 2 files changed, 6 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 3b16e41a9..64f530b53 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -206,7 +206,6 @@ def refresh_data(datadir: Path, pairs: List[str], exchange: Exchange, timerange: Optional[TimeRange] = None, - startup_candles: int = 0, ) -> None: """ Refresh ticker history data for a list of pairs. @@ -216,11 +215,7 @@ def refresh_data(datadir: Path, :param pairs: List of pairs to load :param exchange: Exchange object :param timerange: Limit data to be loaded to this timerange - :param startup_candles: Additional candles to load at the start of the period """ - if startup_candles > 0 and timerange: - logger.info(f'Using indicator startup period: {startup_candles} ...') - for pair in pairs: _download_pair_history(pair=pair, timeframe=timeframe, datadir=datadir, timerange=timerange, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index b52b2ffdc..85029905b 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -101,7 +101,6 @@ class Edge: exchange=self.exchange, timeframe=self.strategy.ticker_interval, timerange=self._timerange, - startup_candles=self.strategy.startup_candle_count, ) data = history.load_data( From c5e6a34f25081498908ef0f44fc7366cf59374f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Dec 2019 19:30:04 +0100 Subject: [PATCH 043/128] Remove unnecessary parenteses --- freqtrade/persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 10896baaa..993b68bc7 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -346,7 +346,7 @@ class Trade(_DECL_BASE): Calculate the open_rate including open_fee. :return: Price in of the open trade incl. Fees """ - buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) + buy_trade = Decimal(self.amount) * Decimal(self.open_rate) fees = buy_trade * Decimal(self.fee_open) return float(buy_trade + fees) @@ -370,7 +370,7 @@ class Trade(_DECL_BASE): if rate is None and not self.close_rate: return 0.0 - sell_trade = (Decimal(self.amount) * Decimal(rate or self.close_rate)) + sell_trade = Decimal(self.amount) * Decimal(rate or self.close_rate) fees = sell_trade * Decimal(fee or self.fee_close) return float(sell_trade - fees) From cf4c3642ce1ea09c758b16a8774cabcea0674ec0 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 18 Dec 2019 01:06:03 +0300 Subject: [PATCH 044/128] Minor improvements in data.history --- freqtrade/data/history.py | 20 ++++++++------------ freqtrade/edge/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/optimize/hyperopt.py | 6 +++--- tests/data/test_converter.py | 4 ++-- tests/data/test_history.py | 15 +++++---------- tests/optimize/test_backtest_detail.py | 4 ++-- tests/optimize/test_backtesting.py | 20 ++++++++++---------- tests/optimize/test_hyperopt.py | 22 +++++++++++----------- 9 files changed, 44 insertions(+), 53 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 64f530b53..4c5c0521f 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -68,7 +68,7 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None) -> Optional[list]: + timerange: Optional[TimeRange] = None) -> List[Dict]: """ Load a pair from file, either .json.gz or .json :return: tickerlist or None if unsuccessful @@ -276,7 +276,7 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, def _download_pair_history(datadir: Path, - exchange: Optional[Exchange], + exchange: Exchange, pair: str, timeframe: str = '5m', timerange: Optional[TimeRange] = None) -> bool: @@ -293,11 +293,6 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ - if not exchange: - raise OperationalException( - "Exchange needs to be initialized when downloading pair history data" - ) - try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' @@ -447,18 +442,19 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], store_tickerdata_file(datadir, pair, timeframe, data=ohlcv) -def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: +def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ - Get the maximum timeframe for the given backtest data + Get the maximum common timerange for the given backtest data. + :param data: dictionary with preprocessed backtesting data :return: tuple containing min_date, max_date """ - timeframe = [ + timeranges = [ (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) for frame in data.values() ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] + return (min(timeranges, key=operator.itemgetter(0))[0], + max(timeranges, key=operator.itemgetter(1))[1]) def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 85029905b..e56071a98 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -120,7 +120,7 @@ class Edge: preprocessed = self.strategy.tickerdata_to_dataframe(data) # Print timeframe - min_date, max_date = history.get_timeframe(preprocessed) + min_date, max_date = history.get_timerange(preprocessed) logger.info( 'Measuring data from %s up to %s (%s days) ...', min_date.isoformat(), diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fc60bd310..726257cdd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -117,7 +117,7 @@ class Backtesting: fail_without_data=True, ) - min_date, max_date = history.get_timeframe(data) + min_date, max_date = history.get_timerange(data) logger.info( 'Loading data from %s up to %s (%s days)..', @@ -481,7 +481,7 @@ class Backtesting: # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = history.trim_dataframe(df, timerange) - min_date, max_date = history.get_timeframe(preprocessed) + min_date, max_date = history.get_timerange(preprocessed) logger.info( 'Backtesting with data from %s up to %s (%s days)..', diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cdde2b722..521a4d790 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,7 +23,7 @@ from joblib import (Parallel, cpu_count, delayed, dump, load, from pandas import DataFrame from freqtrade import OperationalException -from freqtrade.data.history import get_timeframe, trim_dataframe +from freqtrade.data.history import get_timerange, trim_dataframe from freqtrade.misc import plural, round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules @@ -369,7 +369,7 @@ class Hyperopt: processed = load(self.tickerdata_pickle) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) backtesting_results = self.backtesting.backtest( { @@ -490,7 +490,7 @@ class Hyperopt: # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = trim_dataframe(df, timerange) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) logger.info( 'Hyperopting with data from %s up to %s (%s days)..', diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 8184167b3..414551c95 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -2,7 +2,7 @@ import logging from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data -from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timeframe +from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timerange from tests.conftest import log_has @@ -36,7 +36,7 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog): f"{len(data)} - after: {len(data2)}", caplog) # Test fillup actually fixes invalid backtest data - min_date, max_date = get_timeframe({'UNITTEST/BTC': data}) + min_date, max_date = get_timerange({'UNITTEST/BTC': data}) assert validate_backtest_data(data, 'UNITTEST/BTC', min_date, max_date, 1) assert not validate_backtest_data(data2, 'UNITTEST/BTC', min_date, max_date, 1) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index c9b198b39..7b3143db9 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -7,15 +7,13 @@ from shutil import copyfile from unittest.mock import MagicMock, PropertyMock import arrow -import pytest from pandas import DataFrame -from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data.history import (_download_pair_history, _download_trades_history, _load_cached_data_for_updating, - convert_trades_to_ohlcv, get_timeframe, + convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, load_tickerdata_file, pair_data_filename, pair_trades_filename, @@ -138,9 +136,6 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) - with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): - refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], - exchange=None) _clean_test_file(file) @@ -512,7 +507,7 @@ def test_file_dump_json_tofile(testdatadir) -> None: _clean_test_file(file) -def test_get_timeframe(default_conf, mocker, testdatadir) -> None: +def test_get_timerange(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) strategy = DefaultStrategy(default_conf) @@ -523,7 +518,7 @@ def test_get_timeframe(default_conf, mocker, testdatadir) -> None: pairs=['UNITTEST/BTC'] ) ) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' @@ -540,7 +535,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) fill_up_missing=False ) ) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) caplog.clear() assert validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('1m')) @@ -564,7 +559,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No ) ) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) caplog.clear() assert not validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('5m')) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index b36c1b9c5..47cb9f353 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest -from freqtrade.data.history import get_timeframe +from freqtrade.data.history import get_timerange from freqtrade.optimize.backtesting import Backtesting from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange @@ -380,7 +380,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: pair = "UNITTEST/BTC" # Dummy data as we mock the analyze functions data_processed = {pair: frame.copy()} - min_date, max_date = get_timeframe({pair: frame}) + min_date, max_date = get_timerange({pair: frame}) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 38a95be7a..0ea35e41f 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.dataprovider import DataProvider -from freqtrade.data.history import get_timeframe +from freqtrade.data.history import get_timerange from freqtrade.optimize import setup_configuration, start_backtesting from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode @@ -100,7 +100,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: data = load_data_test(contour, testdatadir) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) assert isinstance(processed, dict) results = backtesting.backtest( { @@ -138,7 +138,7 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record= patch_exchange(mocker) backtesting = Backtesting(conf) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) return { 'stake_amount': conf['stake_amount'], 'processed': processed, @@ -458,11 +458,11 @@ def test_generate_text_table_strategyn(default_conf, mocker): def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: - def get_timeframe(input1): + def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) - mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( @@ -491,11 +491,11 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: - def get_timeframe(input1): + def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame())) - mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( @@ -525,7 +525,7 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], timerange=timerange) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(data_processed) + min_date, max_date = get_timerange(data_processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -581,7 +581,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) - data = history.load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'], timerange=timerange) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -701,7 +701,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) backtesting.strategy.advise_sell = _trend_alternate_hold # Override data_processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(data_processed) + min_date, max_date = get_timerange(data_processed) backtest_conf = { 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index c3afa4911..29b8b5b16 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -251,7 +251,7 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame)) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -427,7 +427,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -602,7 +602,7 @@ def test_generate_optimizer(mocker, default_conf) -> None: MagicMock(return_value=backtest_result) ) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13))) ) patch_exchange(mocker) @@ -726,7 +726,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -769,7 +769,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -811,7 +811,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -851,7 +851,7 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -899,7 +899,7 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) - mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -930,7 +930,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -977,7 +977,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -1030,7 +1030,7 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) From 21622ac3138713555784e1ee58cfe61e127fcb40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 16:34:30 +0100 Subject: [PATCH 045/128] Rename get_ticker to fetch_ticker --- freqtrade/exchange/exchange.py | 2 +- freqtrade/freqtradebot.py | 6 +- tests/exchange/test_exchange.py | 14 +-- tests/rpc/test_rpc.py | 36 +++---- tests/rpc/test_rpc_apiserver.py | 12 +-- tests/rpc/test_rpc_telegram.py | 28 +++--- tests/test_freqtradebot.py | 170 ++++++++++++++++---------------- tests/test_integration.py | 4 +- 8 files changed, 136 insertions(+), 136 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e4e7aacce..384ec1fb7 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -524,7 +524,7 @@ class Exchange: raise OperationalException(e) from e @retrier - def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: + def fetch_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: if refresh or pair not in self._cached_ticker.keys(): try: if pair not in self._api.markets or not self._api.markets[pair].get('active'): diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8ae027fa2..ea9b892d7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -192,7 +192,7 @@ class FreqtradeBot: else: if not tick: logger.info('Using Last Ask / Last Price') - ticker = self.exchange.get_ticker(pair) + ticker = self.exchange.fetch_ticker(pair) else: ticker = tick if ticker['ask'] < ticker['last']: @@ -570,7 +570,7 @@ class FreqtradeBot: """ Get sell rate - either using get-ticker bid or first bid based on orderbook The orderbook portion is only used for rpc messaging, which would otherwise fail - for BitMex (has no bid/ask in get_ticker) + for BitMex (has no bid/ask in fetch_ticker) or remain static in any other case since it's not updating. :return: Bid rate """ @@ -582,7 +582,7 @@ class FreqtradeBot: rate = order_book['bids'][0][0] else: - rate = self.exchange.get_ticker(pair, refresh)['bid'] + rate = self.exchange.fetch_ticker(pair, refresh)['bid'] return rate def handle_trade(self, trade: Trade) -> bool: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 629f99aa2..7c273da9f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -977,7 +977,7 @@ def test_get_tickers(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_ticker(default_conf, mocker, exchange_name): +def test_fetch_ticker(default_conf, mocker, exchange_name): api_mock = MagicMock() tick = { 'symbol': 'ETH/BTC', @@ -989,7 +989,7 @@ def test_get_ticker(default_conf, mocker, exchange_name): api_mock.markets = {'ETH/BTC': {'active': True}} exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) # retrieve original ticker - ticker = exchange.get_ticker(pair='ETH/BTC') + ticker = exchange.fetch_ticker(pair='ETH/BTC') assert ticker['bid'] == 0.00001098 assert ticker['ask'] == 0.00001099 @@ -1006,7 +1006,7 @@ def test_get_ticker(default_conf, mocker, exchange_name): # if not caching the result we should get the same ticker # if not fetching a new result we should get the cached ticker - ticker = exchange.get_ticker(pair='ETH/BTC') + ticker = exchange.fetch_ticker(pair='ETH/BTC') assert api_mock.fetch_ticker.call_count == 1 assert ticker['bid'] == 0.5 @@ -1018,19 +1018,19 @@ def test_get_ticker(default_conf, mocker, exchange_name): # Test caching api_mock.fetch_ticker = MagicMock() - exchange.get_ticker(pair='ETH/BTC', refresh=False) + exchange.fetch_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, - "get_ticker", "fetch_ticker", + "fetch_ticker", "fetch_ticker", pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.get_ticker(pair='ETH/BTC', refresh=True) + exchange.fetch_ticker(pair='ETH/BTC', refresh=True) with pytest.raises(DependencyException, match=r'Pair XRP/ETH not available'): - exchange.get_ticker(pair='XRP/ETH', refresh=True) + exchange.fetch_ticker(pair='XRP/ETH', refresh=True) @pytest.mark.parametrize("exchange_name", EXCHANGES) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 0a8c1cabd..3b897572c 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -29,7 +29,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -65,7 +65,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_order': '(limit buy rem=0.00000000)' } == results[0] - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) # invalidate ticker cache rpc._freqtrade.exchange._cached_ticker = {} @@ -104,7 +104,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -134,7 +134,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert 'ETH/BTC' == result[0][1] assert '-0.59% (-0.09)' == result[0][3] - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) # invalidate ticker cache rpc._freqtrade.exchange._cached_ticker = {} @@ -149,7 +149,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -201,7 +201,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -225,7 +225,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -239,7 +239,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -260,7 +260,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) # invalidate ticker cache rpc._freqtrade.exchange._cached_ticker = {} @@ -287,7 +287,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -306,7 +306,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up, + fetch_ticker=ticker_sell_up, get_fee=fee ) trade.update(limit_sell_order) @@ -439,7 +439,7 @@ def test_rpc_start(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock() + fetch_ticker=MagicMock() ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -460,7 +460,7 @@ def test_rpc_stop(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock() + fetch_ticker=MagicMock() ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -482,7 +482,7 @@ def test_rpc_stopbuy(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock() + fetch_ticker=MagicMock() ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -502,7 +502,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, cancel_order=cancel_order_mock, get_order=MagicMock( return_value={ @@ -604,7 +604,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -637,7 +637,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -661,7 +661,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, buy=buy_mm ) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f1e3421c5..36bb81e41 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -256,7 +256,7 @@ def test_api_count(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -292,7 +292,7 @@ def test_api_daily(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -308,7 +308,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -323,7 +323,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -413,7 +413,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -541,7 +541,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index b02f11394..ed2aad37e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -150,7 +150,7 @@ def test_status(default_conf, update, mocker, fee, ticker,) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) msg_mock = MagicMock() @@ -204,7 +204,7 @@ def test_status(default_conf, update, mocker, fee, ticker,) -> None: def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) msg_mock = MagicMock() @@ -254,7 +254,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) @@ -307,7 +307,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) msg_mock = MagicMock() @@ -373,7 +373,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker + fetch_ticker=ticker ) msg_mock = MagicMock() mocker.patch.multiple( @@ -411,7 +411,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) msg_mock = MagicMock() @@ -443,7 +443,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, msg_mock.reset_mock() # Update the ticker with a market going up - mocker.patch('freqtrade.exchange.Exchange.get_ticker', ticker_sell_up) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -700,7 +700,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -715,7 +715,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, assert trade # Increase the price and sell it - mocker.patch('freqtrade.exchange.Exchange.get_ticker', ticker_sell_up) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up) # /forcesell 1 context = MagicMock() @@ -755,7 +755,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -769,7 +769,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) trade = Trade.query.first() @@ -812,7 +812,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) default_conf['max_open_trades'] = 4 @@ -963,7 +963,7 @@ def test_performance_handle(default_conf, update, ticker, fee, ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -998,7 +998,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 341bc021f..13f1277b9 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -157,7 +157,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, patch_wallet(mocker, free=default_conf['stake_amount']) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee ) @@ -232,7 +232,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf buy_price = limit_buy_order['price'] mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price * 0.79, 'ask': buy_price * 0.79, 'last': buy_price * 0.79 @@ -272,7 +272,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, buy_price = limit_buy_order['price'] mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price * 0.85, 'ask': buy_price * 0.85, 'last': buy_price * 0.85 @@ -304,7 +304,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, default_conf['max_open_trades'] = 2 mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -467,7 +467,7 @@ def test_create_trades(default_conf, ticker, limit_buy_order, fee, mocker) -> No patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -501,7 +501,7 @@ def test_create_trades_no_stake_amount(default_conf, ticker, limit_buy_order, patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -519,7 +519,7 @@ def test_create_trades_minimal_amount(default_conf, ticker, limit_buy_order, buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=buy_mock, get_fee=fee, ) @@ -539,7 +539,7 @@ def test_create_trades_too_small_stake_amount(default_conf, ticker, limit_buy_or buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=buy_mock, get_fee=fee, ) @@ -558,7 +558,7 @@ def test_create_trades_limit_reached(default_conf, ticker, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, @@ -579,7 +579,7 @@ def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -600,7 +600,7 @@ def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_ord patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -639,7 +639,7 @@ def test_create_trades_multiple_trades(default_conf, ticker, default_conf['max_open_trades'] = max_open mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': "12355555"}), get_fee=fee, ) @@ -658,7 +658,7 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: default_conf['max_open_trades'] = 4 mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': "12355555"}), get_fee=fee, ) @@ -684,7 +684,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -718,7 +718,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker) -> None: patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(side_effect=TemporaryError) ) sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) @@ -735,7 +735,7 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(side_effect=OperationalException) ) worker = Worker(args=None, config=default_conf) @@ -753,7 +753,7 @@ def test_process_trade_handling(default_conf, ticker, limit_buy_order, fee, mock patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -780,7 +780,7 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -830,7 +830,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: refresh_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(side_effect=TemporaryError), refresh_latest_ohlcv=refresh_mock, ) @@ -853,7 +853,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={'ask': 20, 'last': 10})) assert freqtrade.get_target_bid('ETH/BTC') == 20 @@ -862,7 +862,7 @@ def test_balance_fully_ask_side(mocker, default_conf) -> None: def test_balance_fully_last_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={'ask': 20, 'last': 10})) assert freqtrade.get_target_bid('ETH/BTC') == 10 @@ -871,7 +871,7 @@ def test_balance_fully_last_side(mocker, default_conf) -> None: def test_balance_bigger_last_ask(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={'ask': 5, 'last': 10})) assert freqtrade.get_target_bid('ETH/BTC') == 5 @@ -891,7 +891,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1000,7 +1000,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1100,7 +1100,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1134,7 +1134,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1177,7 +1177,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1231,7 +1231,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, assert freqtrade.handle_stoploss_on_exchange(trade) is False # price jumped 2x - mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00002344, 'ask': 0.00002346, 'last': 0.00002344 @@ -1271,7 +1271,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1342,7 +1342,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, edge_conf['dry_run_wallet'] = 999.9 mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1406,7 +1406,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_order_mock) # price goes down 5% - mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00001172 * 0.95, 'ask': 0.00001173 * 0.95, 'last': 0.00001172 * 0.95 @@ -1422,7 +1422,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, cancel_order_mock.assert_not_called() # price jumped 2x - mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00002344, 'ask': 0.00002346, 'last': 0.00002344 @@ -1662,7 +1662,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1701,7 +1701,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1754,7 +1754,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1786,7 +1786,7 @@ def test_handle_trade_use_sell_signal( patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1814,7 +1814,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1842,7 +1842,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, op patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old), cancel_order=cancel_order_mock, get_fee=fee @@ -1869,7 +1869,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o limit_buy_order_old.update({"status": "canceled"}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old), cancel_order=cancel_order_mock, get_fee=fee @@ -1896,7 +1896,7 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(side_effect=DependencyException), cancel_order=cancel_order_mock, get_fee=fee @@ -1921,7 +1921,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock ) @@ -1949,7 +1949,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock ) @@ -1976,7 +1976,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order=cancel_order_mock ) @@ -2003,7 +2003,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order=cancel_order_mock, get_trades_for_order=MagicMock(return_value=trades_for_order), @@ -2040,7 +2040,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order=cancel_order_mock, get_trades_for_order=MagicMock(return_value=trades_for_order), @@ -2084,7 +2084,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), cancel_order=cancel_order_mock ) @@ -2174,7 +2174,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2190,7 +2190,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) @@ -2222,7 +2222,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2238,7 +2238,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], @@ -2272,7 +2272,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2288,7 +2288,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) default_conf['dry_run'] = True @@ -2330,7 +2330,7 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, c patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, sell=sellmock ) @@ -2367,7 +2367,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke cancel_order = MagicMock(return_value=True) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, symbol_amount_prec=lambda s, x, y: y, symbol_price_prec=lambda s, x, y: y, @@ -2391,7 +2391,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], @@ -2410,7 +2410,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, symbol_amount_prec=lambda s, x, y: y, symbol_price_prec=lambda s, x, y: y, @@ -2474,7 +2474,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2490,7 +2490,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) freqtrade.config['order_types']['sell'] = 'market' @@ -2528,7 +2528,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00002172, 'ask': 0.00002173, 'last': 0.00002172 @@ -2559,7 +2559,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00002172, 'ask': 0.00002173, 'last': 0.00002172 @@ -2588,7 +2588,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00000172, 'ask': 0.00000173, 'last': 0.00000172 @@ -2617,7 +2617,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.0000172, 'ask': 0.0000173, 'last': 0.0000172 @@ -2648,7 +2648,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -2663,7 +2663,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], @@ -2684,7 +2684,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.0000172, 'ask': 0.0000173, 'last': 0.0000172 @@ -2717,7 +2717,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001099, 'ask': 0.00001099, 'last': 0.00001099 @@ -2736,7 +2736,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) assert freqtrade.handle_trade(trade) is False # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00001099 * 1.5, 'ask': 0.00001099 * 1.5, @@ -2747,7 +2747,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) assert freqtrade.handle_trade(trade) is False # Price fell - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00001099 * 1.1, 'ask': 0.00001099 * 1.1, @@ -2771,7 +2771,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price - 0.000001, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 @@ -2795,7 +2795,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, assert freqtrade.handle_trade(trade) is False # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000003, 'ask': buy_price + 0.000003, @@ -2807,7 +2807,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000002, 'ask': buy_price + 0.000002, @@ -2828,7 +2828,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price - 0.000001, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 @@ -2852,7 +2852,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, assert freqtrade.handle_trade(trade) is False # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000003, 'ask': buy_price + 0.000003, @@ -2865,7 +2865,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000002, 'ask': buy_price + 0.000002, @@ -2889,7 +2889,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price, 'ask': buy_price, 'last': buy_price @@ -2916,7 +2916,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, assert trade.stop_loss == 0.0000098910 # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.0000004, 'ask': buy_price + 0.0000004, @@ -2930,7 +2930,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, assert trade.stop_loss == 0.0000098910 # price rises above the offset (rises 12% when the offset is 5.5%) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.0000014, 'ask': buy_price + 0.0000014, @@ -2950,7 +2950,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00000172, 'ask': 0.00000173, 'last': 0.00000172 @@ -3287,7 +3287,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -3324,7 +3324,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -3347,7 +3347,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_order_book=order_book_l2, - get_ticker=ticker_mock, + fetch_ticker=ticker_mock, ) default_conf['exchange']['name'] = 'binance' @@ -3371,7 +3371,7 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_order_book=order_book_l2, - get_ticker=ticker_mock, + fetch_ticker=ticker_mock, ) default_conf['exchange']['name'] = 'binance' @@ -3421,7 +3421,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -3451,7 +3451,7 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_order_book=order_book_l2, - get_ticker=ticker, + fetch_ticker=ticker, ) pair = "ETH/BTC" @@ -3530,7 +3530,7 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) diff --git a/tests/test_integration.py b/tests/test_integration.py index 728e96d55..a80600cdc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -55,7 +55,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, symbol_amount_prec=lambda s, x, y: y, symbol_price_prec=lambda s, x, y: y, @@ -126,7 +126,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc )) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, symbol_amount_prec=lambda s, x, y: y, symbol_price_prec=lambda s, x, y: y, From dc07037edfc2bebed2d3815c986a999d281c5b0e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 16:38:57 +0100 Subject: [PATCH 046/128] Add documentation for price finding --- docs/configuration.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 73534b6f1..bfa29529d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -393,6 +393,41 @@ The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" ``` +## Prices used for orders + +Prices for regular orders can be controlled via the parameter structures `bid_strategy` for Buying, and `ask_strategy` for selling. +Prices are always retrieved right before an order is placed, either by querying the `fetch_ticker()` endpoint of the exchange (usually `/ticker`), or by using the orderbook. + +### Buy price with Orderbook enabled + +When buying with the orderbook enabled (`bid_strategy.use_order_book=True`) - Freqtrade will fetch the `bid_strategy.order_book_top` entries in the orderbook, and will then use the entry specified as `bid_strategy.order_book_top` on the `bids` side of the orderbook. 1 specifies the topmost entry in the Orderbook - while 2 would use the 2nd entry in the Orderbook. + +### Buy price without Orderbook + +When not using orderbook (`bid_strategy.use_order_book=False`), then Freqtrade will use the best `ask` price based on a call to `fetch_ticker()` if it's below the `last` traded price. +Otherwise, it'll use the following formula to calculate the rate: + +``` python +ticker['ask'] + ask_last_balance * (ticker['last'] - ticker['ask']) +``` + +This means - it uses the difference between last and ask (which must be negative, since ask was checked to be higher than last) and multiplies this with `bid_strategy.ask_last_balance` - lowering the price by `balance * (difference between last and ask). + +### Sell price with Orderbook enabled + +When selling with the Orderbook enabled (`ask_strategy.use_order_book=True`) - Freqtrade will fetch the `ask_strategy.order_book_max` entries in the orderbook. Freqtrade will then validate each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the `ask` side for a profitable sell-possibility and will place a sell order at the first profitable spot. + +The idea here is to place the sell-order early, to be ahead in the queue. +A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting `ask_strategy.order_book_min` and `ask_strategy.order_book_max` to the same number. + +!!! Warning "Orderbook and stoploss_on_exchange" + Using `ask_strategy.order_book_max` higher than 1 may increase the risk, since an eventual [stoploss on exchange](#understand-order_types) will be need to be cancelled as soon as the order is placed. + + +### Sell price without orderbook + +When not using orderbook (`ask_strategy.use_order_book=False`), then the best `bid` will be used as sell rate based on a call to `fetch_ticker()`. + ## Pairlists Pairlists define the list of pairs that the bot should trade. From d73ba71ec63d08d9ddc98c983f10aa7509cd9b0b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 16:41:54 +0100 Subject: [PATCH 047/128] Improve formatting of orderbook doc --- docs/configuration.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bfa29529d..e4d29825e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -398,11 +398,13 @@ The valid values are: Prices for regular orders can be controlled via the parameter structures `bid_strategy` for Buying, and `ask_strategy` for selling. Prices are always retrieved right before an order is placed, either by querying the `fetch_ticker()` endpoint of the exchange (usually `/ticker`), or by using the orderbook. -### Buy price with Orderbook enabled +### Buy price + +#### Buy price with Orderbook enabled When buying with the orderbook enabled (`bid_strategy.use_order_book=True`) - Freqtrade will fetch the `bid_strategy.order_book_top` entries in the orderbook, and will then use the entry specified as `bid_strategy.order_book_top` on the `bids` side of the orderbook. 1 specifies the topmost entry in the Orderbook - while 2 would use the 2nd entry in the Orderbook. -### Buy price without Orderbook +#### Buy price without Orderbook When not using orderbook (`bid_strategy.use_order_book=False`), then Freqtrade will use the best `ask` price based on a call to `fetch_ticker()` if it's below the `last` traded price. Otherwise, it'll use the following formula to calculate the rate: @@ -413,7 +415,9 @@ ticker['ask'] + ask_last_balance * (ticker['last'] - ticker['ask']) This means - it uses the difference between last and ask (which must be negative, since ask was checked to be higher than last) and multiplies this with `bid_strategy.ask_last_balance` - lowering the price by `balance * (difference between last and ask). -### Sell price with Orderbook enabled +### Sell price + +#### Sell price with Orderbook enabled When selling with the Orderbook enabled (`ask_strategy.use_order_book=True`) - Freqtrade will fetch the `ask_strategy.order_book_max` entries in the orderbook. Freqtrade will then validate each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the `ask` side for a profitable sell-possibility and will place a sell order at the first profitable spot. @@ -424,7 +428,7 @@ A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting Using `ask_strategy.order_book_max` higher than 1 may increase the risk, since an eventual [stoploss on exchange](#understand-order_types) will be need to be cancelled as soon as the order is placed. -### Sell price without orderbook +#### Sell price without orderbook When not using orderbook (`ask_strategy.use_order_book=False`), then the best `bid` will be used as sell rate based on a call to `fetch_ticker()`. From 1c19856d26c8ea9ed48c293519f8f63a12cbb568 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 16:49:56 +0100 Subject: [PATCH 048/128] add section about depth_of_market --- docs/configuration.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index e4d29825e..5c748b0cb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -400,6 +400,12 @@ Prices are always retrieved right before an order is placed, either by querying ### Buy price +#### Check depth of market + +When enabling `bid_strategy.check_depth_of_market=True`, then buy signals will be filtered based on the orderbook size. + +#TODO: Finish this section + #### Buy price with Orderbook enabled When buying with the orderbook enabled (`bid_strategy.use_order_book=True`) - Freqtrade will fetch the `bid_strategy.order_book_top` entries in the orderbook, and will then use the entry specified as `bid_strategy.order_book_top` on the `bids` side of the orderbook. 1 specifies the topmost entry in the Orderbook - while 2 would use the 2nd entry in the Orderbook. @@ -419,7 +425,7 @@ This means - it uses the difference between last and ask (which must be negative #### Sell price with Orderbook enabled -When selling with the Orderbook enabled (`ask_strategy.use_order_book=True`) - Freqtrade will fetch the `ask_strategy.order_book_max` entries in the orderbook. Freqtrade will then validate each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the `ask` side for a profitable sell-possibility and will place a sell order at the first profitable spot. +When selling with the Orderbook enabled (`ask_strategy.use_order_book=True`) - Freqtrade will fetch the `ask_strategy.order_book_max` entries in the orderbook. Freqtrade will then validate each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the `ask` side for a profitable sell-possibility based on the strategy configuration and will place a sell order at the first profitable spot. The idea here is to place the sell-order early, to be ahead in the queue. A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting `ask_strategy.order_book_min` and `ask_strategy.order_book_max` to the same number. From 11e787c8846027f14e6bc0872743dd5529cf57f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 19:27:54 +0100 Subject: [PATCH 049/128] Finish depth_of_market documentation piece --- docs/configuration.md | 45 ++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5c748b0cb..ba592d436 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -57,12 +57,12 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* | `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* | `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* -| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#understand-ask_last_balance). -| `bid_strategy.use_order_book` | Enable buying using the rates in Order Book Bids.
***Datatype:*** *Boolean* -| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. *Defaults to `1`.*
***Datatype:*** *Positive Integer* -| `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book.
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. *Defaults to `0`.*
***Datatype:*** *Float (as ratio)* -| `ask_strategy.use_order_book` | Enable selling of open trades using Order Book Asks.
***Datatype:*** *Boolean* +| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#buy-price-without-orderbook). +| `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled).
***Datatype:*** *Boolean* +| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled).
*Defaults to `1`.*
***Datatype:*** *Positive Integer* +| `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book. [Check market depth](#check-depth-of-market).
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The difference ratio of buy orders and sell orders found in Order Book. A value below 1 means sell order size is greater, while value greater than 1 means buy order size is higher. [Check market depth](#check-depth-of-market)
*Defaults to `0`.*
***Datatype:*** *Float (as ratio)* +| `ask_strategy.use_order_book` | Enable selling of open trades using [Order Book Asks](#sell-price-with-orderbook-enabled).
***Datatype:*** *Boolean* | `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* | `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* | `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
***Datatype:*** *Boolean* @@ -88,9 +88,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* | `webhook.enabled` | Enable usage of Webhook notifications
***Datatype:*** *Boolean* | `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* | `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Boolean* | `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *IPv4* | `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Integer between 1024 and 65535* @@ -207,13 +207,6 @@ before asking the strategy if we should buy or a sell an asset. After each wait every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or the static list of pairs) if we should buy. -### Understand ask_last_balance - -The `ask_last_balance` configuration parameter sets the bidding price. Value `0.0` will use `ask` price, `1.0` will -use the `last` price and values between those interpolate between ask and last -price. Using `ask` price will guarantee quick success in bid, but bot will also -end up paying more then would probably have been necessary. - ### Understand order_types The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. @@ -398,13 +391,15 @@ The valid values are: Prices for regular orders can be controlled via the parameter structures `bid_strategy` for Buying, and `ask_strategy` for selling. Prices are always retrieved right before an order is placed, either by querying the `fetch_ticker()` endpoint of the exchange (usually `/ticker`), or by using the orderbook. -### Buy price +### Buy price #### Check depth of market -When enabling `bid_strategy.check_depth_of_market=True`, then buy signals will be filtered based on the orderbook size. +When enabling `bid_strategy.check_depth_of_market.enabled=True`, then buy signals will be filtered based on the orderbook size for each size (sum of all amounts). +Orderbook bid size is then divided by Orderbook ask size - and the resulting delta is compared to `bid_strategy.check_depth_of_market.bids_to_ask_delta`, and a buy is only executed if the orderbook delta is bigger or equal to the configured delta. -#TODO: Finish this section +!!! Note: + A calculated delta below 1 means that sell order size is greater, while value greater than 1 means buy order size is higher #### Buy price with Orderbook enabled @@ -413,15 +408,13 @@ When buying with the orderbook enabled (`bid_strategy.use_order_book=True`) - Fr #### Buy price without Orderbook When not using orderbook (`bid_strategy.use_order_book=False`), then Freqtrade will use the best `ask` price based on a call to `fetch_ticker()` if it's below the `last` traded price. -Otherwise, it'll use the following formula to calculate the rate: +Otherwise, it'll calculate a rate between `ask` and `last` price. -``` python -ticker['ask'] + ask_last_balance * (ticker['last'] - ticker['ask']) -``` +The `bid_strategy.ask_last_balance` configuration parameter controls this. A value of `0.0` will use `ask` price, `1.0` will use the `last` price and values between those interpolate between ask and last +price. +Using `ask` price will guarantee quick success in bid, but bot will also end up paying more than what would have been necessary. -This means - it uses the difference between last and ask (which must be negative, since ask was checked to be higher than last) and multiplies this with `bid_strategy.ask_last_balance` - lowering the price by `balance * (difference between last and ask). - -### Sell price +### Sell price #### Sell price with Orderbook enabled From 1af962899d3f443b639d6953f406f145f1cab344 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 19:43:37 +0100 Subject: [PATCH 050/128] Fix note-box syntax error --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index ba592d436..8ae824277 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -398,7 +398,7 @@ Prices are always retrieved right before an order is placed, either by querying When enabling `bid_strategy.check_depth_of_market.enabled=True`, then buy signals will be filtered based on the orderbook size for each size (sum of all amounts). Orderbook bid size is then divided by Orderbook ask size - and the resulting delta is compared to `bid_strategy.check_depth_of_market.bids_to_ask_delta`, and a buy is only executed if the orderbook delta is bigger or equal to the configured delta. -!!! Note: +!!! Note A calculated delta below 1 means that sell order size is greater, while value greater than 1 means buy order size is higher #### Buy price with Orderbook enabled From e72c6a0d94d810b2976c4a6729291fe51602db9d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 20:02:15 +0100 Subject: [PATCH 051/128] use only first part of the currency to get wallet-amount (!!) --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4a48bba04..5656268c8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -903,7 +903,7 @@ class FreqtradeBot: :return: amount to sell :raise: DependencyException: if available balance is not within 2% of the available amount. """ - wallet_amount = self.wallets.get_free(pair) + wallet_amount = self.wallets.get_free(pair.split('/')[0]) logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}") if wallet_amount > amount: return amount From 6507a26cc160fdfe99a58fbd92a8e0a197f4edef Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 20:16:53 +0100 Subject: [PATCH 052/128] Fix some tests after merge --- freqtrade/freqtradebot.py | 2 +- tests/test_freqtradebot.py | 18 ++++++++++++++---- tests/test_integration.py | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5656268c8..a51ab0dbf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -898,7 +898,7 @@ class FreqtradeBot: Should be trade.amount - but will fall back to the available amount if necessary. This should cover cases where get_real_amount() was not able to update the amount for whatever reason. - :param pair: pair - used for logging + :param pair: Pair we're trying to sell :param amount: amount we expect to be available :return: amount to sell :raise: DependencyException: if available balance is not within 2% of the available amount. diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 300d0ad32..cd5c92199 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1682,6 +1682,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock time.sleep(0.01) # Race condition fix trade.update(limit_buy_order) assert trade.is_open is True + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -2549,6 +2550,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.SELL_SIGNAL.value @@ -2579,6 +2581,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.SELL_SIGNAL.value @@ -2639,6 +2642,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.SELL_SIGNAL.value @@ -2676,7 +2680,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, assert trade.amount != amnt -def test__safe_sell_amount(default_conf, caplog, mocker): +def test__safe_sell_amount(default_conf, fee, caplog, mocker): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 @@ -2687,7 +2691,9 @@ def test__safe_sell_amount(default_conf, caplog, mocker): amount=amount, exchange='binance', open_rate=0.245441, - open_order_id="123456" + open_order_id="123456", + fee_open=fee.return_value, + fee_close=fee.return_value, ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2696,7 +2702,7 @@ def test__safe_sell_amount(default_conf, caplog, mocker): assert log_has_re(r'.*Falling back to wallet-amount.', caplog) -def test__safe_sell_amount_error(default_conf, caplog, mocker): +def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 @@ -2707,7 +2713,9 @@ def test__safe_sell_amount_error(default_conf, caplog, mocker): amount=amount, exchange='binance', open_rate=0.245441, - open_order_id="123456" + open_order_id="123456", + fee_open=fee.return_value, + fee_close=fee.return_value, ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2775,6 +2783,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trade) is False @@ -3512,6 +3521,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order time.sleep(0.01) # Race condition fix trade.update(limit_buy_order) + freqtrade.wallets.update() assert trade.is_open is True patch_get_signal(freqtrade, value=(False, True)) diff --git a/tests/test_integration.py b/tests/test_integration.py index 728e96d55..2daf13db6 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -71,7 +71,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) - mocker.patch("freqtrade.wallets.Wallets.get_free", MagicMock(return_value=1)) + mocker.patch("freqtrade.wallets.Wallets.get_free", MagicMock(return_value=1000)) freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True From 0c6b5e01fb4a617d86e1aa9105e1aba960fdd254 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 20:28:36 +0100 Subject: [PATCH 053/128] Try with github-token --- .github/workflows/ci.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6a111944..0e86abc9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,19 +64,16 @@ jobs: pip install -e . - name: Tests - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - COVERALLS_SERVICE_NAME: travis-ci - TRAVIS: "true" run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc + + - name: Coveralls + if: startsWith(matrix.os, 'ubuntu') + env: + COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | # Allow failure for coveralls - # Fake travis environment to get coveralls working correctly - export TRAVIS_PULL_REQUEST="https://github.com/${GITHUB_REPOSITORY}/pull/$(cat $GITHUB_EVENT_PATH | jq -r .number)" - export TRAVIS_BRANCH=${GITHUB_REF#"ref/heads"} - export CI_BRANCH=${GITHUB_REF#"ref/heads"} - echo "${TRAVIS_BRANCH}" - coveralls || true + coveralls -v || true - name: Backtesting run: | From 342f3f450b1a201b155994f61ce182cc31721db3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Dec 2019 20:38:21 +0100 Subject: [PATCH 054/128] try with coveralls token in yml ... --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e86abc9b..53b2e5440 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,8 @@ jobs: - name: Coveralls if: startsWith(matrix.os, 'ubuntu') env: - COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Coveralls token. Not used as secret due to github not providing secrets to forked repositories + COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu run: | # Allow failure for coveralls coveralls -v || true From fc5764f9df7c4341c9dd14560822ac4eec7fb6cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Dec 2019 19:55:21 +0100 Subject: [PATCH 055/128] Edge small cleanup --- freqtrade/optimize/edge_cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index a667ebb92..3848623db 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -42,11 +42,9 @@ class EdgeCli: # Set refresh_pairs to false for edge-cli (it must be true for edge) self.edge._refresh_pairs = False - self.timerange = TimeRange.parse_timerange(None if self.config.get( + self.edge._timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - self.edge._timerange = self.timerange - def _generate_edge_table(self, results: dict) -> str: floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', '.d') From 95bd9e8e0b5785031e70bedd0295cc763b54d541 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 22 Dec 2019 00:17:51 +0300 Subject: [PATCH 056/128] No underscores in cli options --- docs/bot-usage.md | 40 ++++++++++++++---------- freqtrade/configuration/cli_options.py | 8 ++--- freqtrade/configuration/configuration.py | 4 +-- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 25818aea6..86a990946 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -192,8 +192,8 @@ Backtesting also uses the config specified via `-c/--config`. usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TICKER_INTERVAL] - [--timerange TIMERANGE] [--max_open_trades INT] - [--stake_amount STAKE_AMOUNT] [--fee FLOAT] + [--timerange TIMERANGE] [--max-open-trades INT] + [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--eps] [--dmmp] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] @@ -205,10 +205,12 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. - --max_open_trades INT - Specify max_open_trades to use. - --stake_amount STAKE_AMOUNT - Specify stake_amount. + --max-open-trades INT + Override the value of the `max_open_trades` + configuration setting. + --stake-amount STAKE_AMOUNT + Override the value of the `stake_amount` configuration + setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). --eps, --enable-position-stacking @@ -270,8 +272,8 @@ to find optimal parameter values for your stategy. usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TICKER_INTERVAL] [--timerange TIMERANGE] - [--max_open_trades INT] - [--stake_amount STAKE_AMOUNT] [--fee FLOAT] + [--max-open-trades INT] + [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--hyperopt NAME] [--hyperopt-path PATH] [--eps] [-e INT] [--spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] @@ -286,10 +288,12 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. - --max_open_trades INT - Specify max_open_trades to use. - --stake_amount STAKE_AMOUNT - Specify stake_amount. + --max-open-trades INT + Override the value of the `max_open_trades` + configuration setting. + --stake-amount STAKE_AMOUNT + Override the value of the `stake_amount` configuration + setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). --hyperopt NAME Specify hyperopt class name which will be used by the @@ -360,7 +364,7 @@ To know your trade expectancy and winrate against historical data, you can use E usage: freqtrade edge [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TICKER_INTERVAL] [--timerange TIMERANGE] - [--max_open_trades INT] [--stake_amount STAKE_AMOUNT] + [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--stoplosses STOPLOSS_RANGE] optional arguments: @@ -370,10 +374,12 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. - --max_open_trades INT - Specify max_open_trades to use. - --stake_amount STAKE_AMOUNT - Specify stake_amount. + --max-open-trades INT + Override the value of the `max_open_trades` + configuration setting. + --stake-amount STAKE_AMOUNT + Override the value of the `stake_amount` configuration + setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). --stoplosses STOPLOSS_RANGE diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 30902dfe9..4b6429f20 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -118,14 +118,14 @@ AVAILABLE_CLI_OPTIONS = { help='Specify what timerange of data to use.', ), "max_open_trades": Arg( - '--max_open_trades', - help='Specify max_open_trades to use.', + '--max-open-trades', + help='Override the value of the `max_open_trades` configuration setting.', type=int, metavar='INT', ), "stake_amount": Arg( - '--stake_amount', - help='Specify stake_amount.', + '--stake-amount', + help='Override the value of the `stake_amount` configuration setting.', type=float, ), # Backtesting diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e517a0558..001e89303 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -223,13 +223,13 @@ class Configuration: logger.info('max_open_trades set to unlimited ...') elif 'max_open_trades' in self.args and self.args["max_open_trades"]: config.update({'max_open_trades': self.args["max_open_trades"]}) - logger.info('Parameter --max_open_trades detected, ' + logger.info('Parameter --max-open-trades detected, ' 'overriding max_open_trades to: %s ...', config.get('max_open_trades')) elif config['runmode'] in NON_UTIL_MODES: logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) self._args_to_config(config, argname='stake_amount', - logstring='Parameter --stake_amount detected, ' + logstring='Parameter --stake-amount detected, ' 'overriding stake_amount to: {} ...') self._args_to_config(config, argname='fee', From 9ec4368c6f9cee11cbc72ca5f518fcff57859b02 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 09:20:42 +0100 Subject: [PATCH 057/128] Add release documentation --- docs/developer.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/developer.md b/docs/developer.md index 5b07aff03..c679b8a49 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -266,4 +266,29 @@ Once the PR against master is merged (best right after merging): * Use the button "Draft a new release" in the Github UI (subsection releases). * Use the version-number specified as tag. * Use "master" as reference (this step comes after the above PR is merged). -* Use the above changelog as release comment (as codeblock). +* Use the above changelog as release comment (as codeblock) + +### After-release + +* Update version in develop by postfixing that with `-dev` (`2019.6 -> 2019.6-dev`). +* Create a PR against develop to update that branch. + +## Releases + +### pypi + +To create a pypi release, please run the following commands: + +Additional requirement: `wheel`, `twine` (for uploading), account on pypi with proper permissions. + +``` bash +python setup.py sdist bdist_wheel + +# For pypi test (to check if some change to the installation did work) +twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +# For production: +twine upload dist/* +``` + +Please don't push non-releases to the productive / real pypi instance. From c417877eb8d6ad8cc63f9b7585846837f872b701 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2019 13:41:54 +0200 Subject: [PATCH 058/128] sort pytest dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3710bcdc0..29f9db10c 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ setup(name='freqtrade', license='GPLv3', packages=['freqtrade'], setup_requires=['pytest-runner', 'numpy'], - tests_require=['pytest', 'pytest-mock', 'pytest-cov'], + tests_require=['pytest', 'pytest-asyncio', 'pytest-cov', 'pytest-mock', ], install_requires=[ # from requirements-common.txt 'ccxt>=1.18.1080', From 1a73159200278edc3de3f484811c0ecde6dbaaa9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 06:43:22 +0200 Subject: [PATCH 059/128] Modify classifiers --- setup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 29f9db10c..7d8d7b68d 100644 --- a/setup.py +++ b/setup.py @@ -99,8 +99,12 @@ setup(name='freqtrade', ], }, classifiers=[ - 'Programming Language :: Python :: 3.6', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Topic :: Office/Business :: Financial :: Investment', + 'Environment :: Console', 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Operating System :: MacOS', + 'Operating System :: Unix', + 'Topic :: Office/Business :: Financial :: Investment', ]) From 1ff0d0f1fa05137560475d7c209768fb30d3aefc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 09:35:06 +0100 Subject: [PATCH 060/128] Add unfilledtimeout to strategy overrides --- docs/configuration.md | 5 +++-- freqtrade/resolvers/strategy_resolver.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 73534b6f1..846c4510d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -55,8 +55,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float* | `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
***Datatype:*** *Float* | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* -| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* +| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Integer* +| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Integer* | `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#understand-ask_last_balance). | `bid_strategy.use_order_book` | Enable buying using the rates in Order Book Bids.
***Datatype:*** *Boolean* | `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. *Defaults to `1`.*
***Datatype:*** *Positive Integer* @@ -124,6 +124,7 @@ Values set in the configuration file always overwrite values set in the strategy * `order_time_in_force` * `stake_currency` * `stake_amount` +* `unfilledtimeout` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 9a76b9b74..a2d14fbf3 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -61,6 +61,7 @@ class StrategyResolver(IResolver): ("stake_currency", None, False), ("stake_amount", None, False), ("startup_candle_count", None, False), + ("unfilledtimeout", None, False), ("use_sell_signal", True, True), ("sell_profit_only", False, True), ("ignore_roi_if_buy_signal", False, True), From 9835312033a300dce9bae1e92381b5bcffbdfa48 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 09:46:00 +0100 Subject: [PATCH 061/128] Improve pair_lock handling --- freqtrade/strategy/interface.py | 15 ++++++++++++++- tests/strategy/test_interface.py | 13 +++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 985ff37de..4f2e990d2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -168,11 +168,24 @@ class IStrategy(ABC): """ Locks pair until a given timestamp happens. Locked pairs are not analyzed, and are prevented from opening new trades. + Locks can only count up (allowing users to lock pairs for a longer period of time). + To remove a lock from a pair, use `unlock_pair()` :param pair: Pair to lock :param until: datetime in UTC until the pair should be blocked from opening new trades. Needs to be timezone aware `datetime.now(timezone.utc)` """ - self._pair_locked_until[pair] = until + if pair not in self._pair_locked_until or self._pair_locked_until[pair] < until: + self._pair_locked_until[pair] = until + + def unlock_pair(self, pair) -> None: + """ + Unlocks a pair previously locked using lock_pair. + Not used by freqtrade itself, but intended to be used if users lock pairs + manually from within the strategy, to allow an easy way to unlock pairs. + :param pair: Unlock pair to allow trading again + """ + if pair in self._pair_locked_until: + del self._pair_locked_until[pair] def is_pair_locked(self, pair: str) -> bool: """ diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 605622b8f..89c38bda1 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -302,6 +302,19 @@ def test_is_pair_locked(default_conf): # ETH/BTC locked for 4 minutes assert strategy.is_pair_locked(pair) + # Test lock does not change + lock = strategy._pair_locked_until[pair] + strategy.lock_pair(pair, arrow.utcnow().shift(minutes=2).datetime) + assert lock == strategy._pair_locked_until[pair] + # XRP/BTC should not be locked now pair = 'XRP/BTC' assert not strategy.is_pair_locked(pair) + + # Unlocking a pair that's not locked should not raise an error + strategy.unlock_pair(pair) + + # Unlock original pair + pair = 'ETH/BTC' + strategy.unlock_pair(pair) + assert not strategy.is_pair_locked(pair) From 89b4f45fe353773cafe09fb9a18a941a7b1cfb9b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 09:47:37 +0100 Subject: [PATCH 062/128] Remove section about strategy template - use new-strategy intead --- docs/strategy-customization.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 4efca7d2f..c4a477f80 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -479,11 +479,6 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: Printing more than a few rows is also possible (simply use `print(dataframe)` instead of `print(dataframe.tail())`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds). -### Where can i find a strategy template? - -The strategy template is located in the file -[user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_strategy.py). - ### Specify custom strategy location If you want to use a strategy from a different directory you can pass `--strategy-path` From a71deeda94f4c55ae1a8f30d10d034ffb0acbcd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 09:55:40 +0100 Subject: [PATCH 063/128] Document lock-pair implementation --- docs/strategy-customization.md | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index c4a477f80..011f64d70 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -455,6 +455,49 @@ Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of !!! Warning Trade history is not available during backtesting or hyperopt. +### Prevent trades from happening for a specific pair + +Freqtrade locks pairs automatically for the current candle (until that candle is over) when a pair is sold, preventing an immediate re-buy of that pair. + +Locked pairs will show the message `Pair is currently locked.`. + +#### Locking pairs from within the strategy + +Sometimes it may be desired to lock a pair after certain events happen (e.g. multiple losing trades in a row). + +Freqtrade has an easy method to do this from within the strategy, by calling `self.lock_pair(pair, until)`. +Until should be a time in the future, after which trading will be reenabled for that pair. + +Locks can also be lifted manually, by calling `self.unlock_pair(pair)`. + +!!! Note + Locked pairs are not persisted, so a restart of the bot, or calling `/reload_conf` will reset locked pairs. + +!!! Warning + Locking pairs is not functional during backtesting. + +##### Pair locking example + +``` python +from freqtrade.persistence import Trade +from datetime import timedelta, datetime, timezone +# Put the above lines a the top of the strategy file, next to all the other imports +# -------- + +# Within populate indicators (or populate_buy): +if self.config['runmode'] in ('live', 'dry_run'): + # fetch closed trades for the last 2 days + trades = Trade.get_trades([Trade.pair == metadata['pair'], + Trade.open_date > datetime.utcnow() - timedelta(days=2), + Trade.is_open == False, + ]).all() + # Analyze the conditions you'd like to lock the pair .... will probably be different for every strategy + sumprofit = sum(trade.close_profit for trade in trades) + if sumprofit < 0: + # Lock pair for 2 days + self.lock_pair(metadata['pair'], until=datetime.now(timezone.utc) + timedelta(days=2)) +``` + ### Print created dataframe To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`. From 43c25c8a328edf307dc979be0ae88a3cd3d508f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 09:59:25 +0100 Subject: [PATCH 064/128] add documentation for is_pair_locked --- docs/strategy-customization.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 011f64d70..129939b25 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -466,10 +466,12 @@ Locked pairs will show the message `Pair is currently locked.`. Sometimes it may be desired to lock a pair after certain events happen (e.g. multiple losing trades in a row). Freqtrade has an easy method to do this from within the strategy, by calling `self.lock_pair(pair, until)`. -Until should be a time in the future, after which trading will be reenabled for that pair. +`until` must be a datetime object in the future, after which trading will be reenabled for that pair. Locks can also be lifted manually, by calling `self.unlock_pair(pair)`. +To verify if a pair is currently locked, use `self.is_pair_locked(pair)`. + !!! Note Locked pairs are not persisted, so a restart of the bot, or calling `/reload_conf` will reset locked pairs. From ffd7034c005ead82a79e4a60e20e23e225953141 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 10:16:16 +0100 Subject: [PATCH 065/128] Persist dry-run trade per default --- freqtrade/constants.py | 2 +- tests/test_persistence.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5c7190b41..d7c6249d5 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -10,7 +10,7 @@ HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' -DEFAULT_DB_DRYRUN_URL = 'sqlite://' +DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' DEFAULT_AMOUNT_RESERVE_PERCENT = 0.05 REQUIRED_ORDERTIF = ['buy', 'sell'] diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 25ad8b6a7..b9a636e1a 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -100,7 +100,7 @@ def test_init_dryrun_db(default_conf, mocker): init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 - assert create_engine_mock.mock_calls[0][1][0] == 'sqlite://' + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.dryrun.sqlite' @pytest.mark.usefixtures("init_persistence") From dc567f99d67e5a82026fd3a5ff3002323ac51b24 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 10:16:49 +0100 Subject: [PATCH 066/128] Update documentation to new handling of dry-mode database --- docs/bot-usage.md | 8 +++-- docs/configuration.md | 2 +- docs/docker.md | 3 +- docs/plotting.md | 72 +++++++++++++++++-------------------------- 4 files changed, 37 insertions(+), 48 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 86a990946..e856755d2 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -45,14 +45,17 @@ optional arguments: -h, --help show this help message and exit --db-url PATH Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite` for - Live Run mode, `sqlite://` for Dry Run). + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). --sd-notify Notify systemd service manager. --dry-run Enforce dry-run for trading (removes Exchange secrets and simulates trades). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH Specify configuration file (default: `config.json`). @@ -68,6 +71,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. + ``` ### How to specify which configuration file be used? diff --git a/docs/configuration.md b/docs/configuration.md index 73534b6f1..fbd4c9815 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -96,7 +96,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Integer between 1024 and 65535* | `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* | `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
***Datatype:*** *String, SQLAlchemy connect string* +| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
***Datatype:*** *String, SQLAlchemy connect string* | `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
***Datatype:*** *Enum, either `stopped` or `running`* | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
***Datatype:*** *Boolean* | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
***Datatype:*** *ClassName* diff --git a/docs/docker.md b/docs/docker.md index ff5bf7f25..d1684abc5 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -164,8 +164,7 @@ docker run -d \ ``` !!! Note - db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. - To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` + When using docker, it's best to specify `--db-url` explicitly to ensure that the database URL and the mounted database file match. !!! Note All available bot command line parameters can be added to the end of the `docker run` command. diff --git a/docs/plotting.md b/docs/plotting.md index 982a5cd65..ba737562f 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -23,58 +23,43 @@ The `freqtrade plot-dataframe` subcommand shows an interactive graph with three Possible arguments: ``` -usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] [-s NAME] - [--strategy-path PATH] [-p PAIRS [PAIRS ...]] - [--indicators1 INDICATORS1 [INDICATORS1 ...]] - [--indicators2 INDICATORS2 [INDICATORS2 ...]] - [--plot-limit INT] [--db-url PATH] - [--trade-source {DB,file}] [--export EXPORT] - [--export-filename PATH] - [--timerange TIMERANGE] [-i TICKER_INTERVAL] +usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] + [--strategy-path PATH] [-p PAIRS [PAIRS ...]] [--indicators1 INDICATORS1 [INDICATORS1 ...]] + [--indicators2 INDICATORS2 [INDICATORS2 ...]] [--plot-limit INT] [--db-url PATH] + [--trade-source {DB,file}] [--export EXPORT] [--export-filename PATH] [--timerange TIMERANGE] + [-i TICKER_INTERVAL] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space- - separated. + Show profits for only these pairs. Pairs are space-separated. --indicators1 INDICATORS1 [INDICATORS1 ...] - Set indicators from your strategy you want in the - first row of the graph. Space-separated list. Example: + Set indicators from your strategy you want in the first row of the graph. Space-separated list. Example: `ema3 ema5`. Default: `['sma', 'ema3', 'ema5']`. --indicators2 INDICATORS2 [INDICATORS2 ...] - Set indicators from your strategy you want in the - third row of the graph. Space-separated list. Example: + Set indicators from your strategy you want in the third row of the graph. Space-separated list. Example: `fastd fastk`. Default: `['macd', 'macdsignal']`. - --plot-limit INT Specify tick limit for plotting. Notice: too high - values cause huge files. Default: 750. - --db-url PATH Override trades database URL, this is useful in custom - deployments (default: `sqlite:///tradesv3.sqlite` for - Live Run mode, `sqlite://` for Dry Run). + --plot-limit INT Specify tick limit for plotting. Notice: too high values cause huge files. Default: 750. + --db-url PATH Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite` + for Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for Dry Run). --trade-source {DB,file} - Specify the source for trades (Can be DB or file - (backtest file)) Default: file - --export EXPORT Export backtest results, argument are: trades. - Example: `--export=trades` + Specify the source for trades (Can be DB or file (backtest file)) Default: file + --export EXPORT Export backtest results, argument are: trades. Example: `--export=trades` --export-filename PATH - Save backtest results to the file with this filename - (default: `user_data/backtest_results/backtest- - result.json`). Requires `--export` to be set as well. - Example: `--export-filename=user_data/backtest_results - /backtest_today.json` + Save backtest results to the file with this filename. Requires `--export` to be set as well. Example: + `--export-filename=user_data/backtest_results/backtest_today.json` --timerange TIMERANGE Specify what timerange of data to use. -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL - Specify ticker interval (`1m`, `5m`, `30m`, `1h`, - `1d`). + Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. @@ -83,8 +68,7 @@ Common arguments: Strategy arguments: -s NAME, --strategy NAME - Specify strategy class name (default: - `DefaultStrategy`). + Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. ``` @@ -173,14 +157,14 @@ optional arguments: --export EXPORT Export backtest results, argument are: trades. Example: `--export=trades` --export-filename PATH - Save backtest results to the file with this filename - (default: `user_data/backtest_results/backtest- - result.json`). Requires `--export` to be set as well. - Example: `--export-filename=user_data/backtest_results - /backtest_today.json` + Save backtest results to the file with this filename. + Requires `--export` to be set as well. Example: + `--export-filename=user_data/backtest_results/backtest + _today.json` --db-url PATH Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite` for - Live Run mode, `sqlite://` for Dry Run). + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). --trade-source {DB,file} Specify the source for trades (Can be DB or file (backtest file)) Default: file @@ -190,7 +174,9 @@ optional arguments: Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH Specify configuration file (default: `config.json`). From 2195ae59d6227de123a85a1a547be0d956b49656 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 12:49:01 +0100 Subject: [PATCH 067/128] Use different time offsets to avoid confusion --- docs/strategy-customization.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 129939b25..d59b097d7 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -476,7 +476,7 @@ To verify if a pair is currently locked, use `self.is_pair_locked(pair)`. Locked pairs are not persisted, so a restart of the bot, or calling `/reload_conf` will reset locked pairs. !!! Warning - Locking pairs is not functional during backtesting. + Locking pairs is not functioning during backtesting. ##### Pair locking example @@ -496,8 +496,8 @@ if self.config['runmode'] in ('live', 'dry_run'): # Analyze the conditions you'd like to lock the pair .... will probably be different for every strategy sumprofit = sum(trade.close_profit for trade in trades) if sumprofit < 0: - # Lock pair for 2 days - self.lock_pair(metadata['pair'], until=datetime.now(timezone.utc) + timedelta(days=2)) + # Lock pair for 12 hours + self.lock_pair(metadata['pair'], until=datetime.now(timezone.utc) + timedelta(hours=12)) ``` ### Print created dataframe From 76a93fabc7027f80b5a45b6365b36891d127d1aa Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 06:26:13 +0000 Subject: [PATCH 068/128] Bump python from 3.7.5-slim-stretch to 3.7.6-slim-stretch Bumps python from 3.7.5-slim-stretch to 3.7.6-slim-stretch. Signed-off-by: dependabot-preview[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dc9b04403..f631d891d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.5-slim-stretch +FROM python:3.7.6-slim-stretch RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ From 8f17b81329ea59ab7ad45a89501b9f42f79b32f4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 07:37:04 +0000 Subject: [PATCH 069/128] Bump ccxt from 1.20.84 to 1.21.12 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.20.84 to 1.21.12. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.20.84...1.21.12) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index a6d9a6f5e..ed6d68742 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.20.84 +ccxt==1.21.12 SQLAlchemy==1.3.11 python-telegram-bot==12.2.0 arrow==0.15.4 From 20ad8a379d51f29d3e00a1f7c3a8cdfbf33ed64b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 07:38:05 +0000 Subject: [PATCH 070/128] Bump numpy from 1.17.4 to 1.18.0 Bumps [numpy](https://github.com/numpy/numpy) from 1.17.4 to 1.18.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.17.4...v1.18.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ebf27abd4..e0e2942b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Load common requirements -r requirements-common.txt -numpy==1.17.4 +numpy==1.18.0 pandas==0.25.3 From 31a7e9feedb72675845f8e1e1cf2fb29e91d8bc7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 07:38:10 +0000 Subject: [PATCH 071/128] Bump mypy from 0.750 to 0.761 Bumps [mypy](https://github.com/python/mypy) from 0.750 to 0.761. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.750...v0.761) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index fe5b4e369..1357bba00 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ coveralls==1.9.2 flake8==3.7.9 flake8-type-annotations==0.1.0 flake8-tidy-imports==3.1.0 -mypy==0.750 +mypy==0.761 pytest==5.3.2 pytest-asyncio==0.10.0 pytest-cov==2.8.1 From 9cfbe98a237549bef2c3d3e87e2d6a933911b8b0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 07:39:25 +0000 Subject: [PATCH 072/128] Bump scipy from 1.3.3 to 1.4.1 Bumps [scipy](https://github.com/scipy/scipy) from 1.3.3 to 1.4.1. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.3.3...v1.4.1) Signed-off-by: dependabot-preview[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b2428e37d..9b408dbed 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.3.3 +scipy==1.4.1 scikit-learn==0.22 scikit-optimize==0.5.2 filelock==3.0.12 From 779278ed507d1f7f9070ee50f8349b720b499869 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 08:28:05 +0000 Subject: [PATCH 073/128] Bump sqlalchemy from 1.3.11 to 1.3.12 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.3.11 to 1.3.12. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index ed6d68742..9d0fd4756 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,7 +1,7 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs ccxt==1.21.12 -SQLAlchemy==1.3.11 +SQLAlchemy==1.3.12 python-telegram-bot==12.2.0 arrow==0.15.4 cachetools==4.0.0 From 1c5f8070e547d3c1766779697e553efa47742d42 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 09:53:55 +0100 Subject: [PATCH 074/128] Refactor build_paths to staticmethod --- freqtrade/resolvers/iresolver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 3bad42fd9..0b986debb 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -17,7 +17,8 @@ class IResolver: This class contains all the logic to load custom classes """ - def build_search_paths(self, config, current_path: Path, user_subdir: Optional[str] = None, + @staticmethod + def build_search_paths(config, current_path: Path, user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: abs_paths: List[Path] = [current_path] From 5fefa9e97c01b83accd9ddd8c44bad83f331304d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 09:56:12 +0100 Subject: [PATCH 075/128] Convert PairlistResolver to static loader --- freqtrade/pairlist/pairlistmanager.py | 14 ++++---- freqtrade/resolvers/pairlist_resolver.py | 43 ++++++++++++++---------- tests/pairlist/test_pairlist.py | 2 +- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index fa5382c37..1530710d2 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -28,13 +28,13 @@ class PairListManager(): if 'method' not in pl: logger.warning(f"No method in {pl}") continue - pairl = PairListResolver(pl.get('method'), - exchange=exchange, - pairlistmanager=self, - config=config, - pairlistconfig=pl, - pairlist_pos=len(self._pairlists) - ).pairlist + pairl = PairListResolver.load_pairlist(pl.get('method'), + exchange=exchange, + pairlistmanager=self, + config=config, + pairlistconfig=pl, + pairlist_pos=len(self._pairlists) + ) self._tickers_needed = pairl.needstickers or self._tickers_needed self._pairlists.append(pairl) diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index d849f4ffb..5b5bcee3a 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -18,23 +18,32 @@ class PairListResolver(IResolver): This class contains all the logic to load custom PairList class """ - __slots__ = ['pairlist'] + __slots__ = [] - def __init__(self, pairlist_name: str, exchange, pairlistmanager, - config: dict, pairlistconfig: dict, pairlist_pos: int) -> None: + @staticmethod + def load_pairlist(pairlist_name: str, exchange, pairlistmanager, + config: dict, pairlistconfig: dict, pairlist_pos: int) -> IPairList: """ - Load the custom class from config parameter - :param config: configuration dictionary or None + Load the pairlist with pairlist_name + :param pairlist_name: Classname of the pairlist + :param exchange: Initialized exchange class + :param pairlistmanager: Initialized pairlist manager + :param config: configuration dictionary + :param pairlistconfig: Configuration dedicated to this pairlist + :param pairlist_pos: Position of the pairlist in the list of pairlists + :return: initialized Pairlist class """ - self.pairlist = self._load_pairlist(pairlist_name, config, - kwargs={'exchange': exchange, - 'pairlistmanager': pairlistmanager, - 'config': config, - 'pairlistconfig': pairlistconfig, - 'pairlist_pos': pairlist_pos}) - def _load_pairlist( - self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList: + return PairListResolver._load_pairlist(pairlist_name, config, + kwargs={'exchange': exchange, + 'pairlistmanager': pairlistmanager, + 'config': config, + 'pairlistconfig': pairlistconfig, + 'pairlist_pos': pairlist_pos}) + + + @staticmethod + def _load_pairlist(pairlist_name: str, config: dict, kwargs: dict) -> IPairList: """ Search and loads the specified pairlist. :param pairlist_name: name of the module to import @@ -44,11 +53,11 @@ class PairListResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=None, extra_dir=None) + abs_paths = IResolver.build_search_paths(config, current_path=current_path, + user_subdir=None, extra_dir=None) - pairlist = self._load_object(paths=abs_paths, object_type=IPairList, - object_name=pairlist_name, kwargs=kwargs) + pairlist = IResolver._load_object(paths=abs_paths, object_type=IPairList, + object_name=pairlist_name, kwargs=kwargs) if pairlist: return pairlist raise OperationalException( diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 43285cdb1..e7f098777 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -53,7 +53,7 @@ def test_load_pairlist_noexist(mocker, markets, default_conf): with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): - PairListResolver('NonexistingPairList', bot.exchange, plm, default_conf, {}, 1) + PairListResolver.load_pairlist('NonexistingPairList', bot.exchange, plm, default_conf, {}, 1) def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): From 560acb7cea6fa34bcf1eb0473cc373970fb97fd0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 10:03:18 +0100 Subject: [PATCH 076/128] Convert ExchangeResolver to static loader class --- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/resolvers/exchange_resolver.py | 20 +++++++++++--------- freqtrade/resolvers/pairlist_resolver.py | 3 --- freqtrade/utils.py | 8 ++++---- tests/conftest.py | 2 +- tests/exchange/test_exchange.py | 8 ++++---- tests/pairlist/test_pairlist.py | 3 ++- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8ae027fa2..001b7f02d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -60,7 +60,7 @@ class FreqtradeBot: # Check config consistency here since strategies can set certain options validate_config_consistency(config) - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) persistence.init(self.config.get('db_url', None), clean_open_orders=self.config.get('dry_run', False)) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 726257cdd..bab997cb1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -60,7 +60,7 @@ class Backtesting: # Reset keys for backtesting remove_credentials(self.config) self.strategylist: List[IStrategy] = [] - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) if config.get('fee'): self.fee = config['fee'] diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index 60f37b1c9..e28a5cf80 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -15,9 +15,8 @@ class ExchangeResolver(IResolver): This class contains all the logic to load a custom exchange class """ - __slots__ = ['exchange'] - - def __init__(self, exchange_name: str, config: dict, validate: bool = True) -> None: + @staticmethod + def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: """ Load the custom class from config parameter :param config: configuration dictionary @@ -25,17 +24,20 @@ class ExchangeResolver(IResolver): # Map exchange name to avoid duplicate classes for identical exchanges exchange_name = MAP_EXCHANGE_CHILDCLASS.get(exchange_name, exchange_name) exchange_name = exchange_name.title() + exchange = None try: - self.exchange = self._load_exchange(exchange_name, kwargs={'config': config, - 'validate': validate}) + exchange = ExchangeResolver._load_exchange(exchange_name, + kwargs={'config': config, + 'validate': validate}) except ImportError: logger.info( f"No {exchange_name} specific subclass found. Using the generic class instead.") - if not hasattr(self, "exchange"): - self.exchange = Exchange(config, validate=validate) + if not exchange: + exchange = Exchange(config, validate=validate) + return exchange - def _load_exchange( - self, exchange_name: str, kwargs: dict) -> Exchange: + @staticmethod + def _load_exchange(exchange_name: str, kwargs: dict) -> Exchange: """ Loads the specified exchange. Only checks for exchanges exported in freqtrade.exchanges diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 5b5bcee3a..611660ff4 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -18,8 +18,6 @@ class PairListResolver(IResolver): This class contains all the logic to load custom PairList class """ - __slots__ = [] - @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, config: dict, pairlistconfig: dict, pairlist_pos: int) -> IPairList: @@ -41,7 +39,6 @@ class PairListResolver(IResolver): 'pairlistconfig': pairlistconfig, 'pairlist_pos': pairlist_pos}) - @staticmethod def _load_pairlist(pairlist_name: str, config: dict, kwargs: dict) -> IPairList: """ diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 9e01c7ea6..18966c574 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -198,7 +198,7 @@ def start_download_data(args: Dict[str, Any]) -> None: pairs_not_available: List[str] = [] # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config).exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) try: if config.get('download_trades'): @@ -233,7 +233,7 @@ def start_list_timeframes(args: Dict[str, Any]) -> None: config['ticker_interval'] = None # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) if args['print_one_column']: print('\n'.join(exchange.timeframes)) @@ -252,7 +252,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) # By default only active pairs/markets are to be shown active_only = not args.get('list_pairs_all', False) @@ -333,7 +333,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: from freqtrade.pairlist.pairlistmanager import PairListManager config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) quote_currencies = args.get('quote_currencies') if not quote_currencies: diff --git a/tests/conftest.py b/tests/conftest.py index 82111528e..501f89fff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,7 +77,7 @@ def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', patch_exchange(mocker, api_mock, id, mock_markets) config["exchange"]["name"] = id try: - exchange = ExchangeResolver(id, config).exchange + exchange = ExchangeResolver.load_exchange(id, config) except ImportError: exchange = Exchange(config) return exchange diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 629f99aa2..dccf7d74f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -124,19 +124,19 @@ def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - exchange = ExchangeResolver('Bittrex', default_conf).exchange + exchange = ExchangeResolver.load_exchange('Bittrex', default_conf) assert isinstance(exchange, Exchange) assert log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog) caplog.clear() - exchange = ExchangeResolver('kraken', default_conf).exchange + exchange = ExchangeResolver.load_exchange('kraken', default_conf) assert isinstance(exchange, Exchange) assert isinstance(exchange, Kraken) assert not isinstance(exchange, Binance) assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog) - exchange = ExchangeResolver('binance', default_conf).exchange + exchange = ExchangeResolver.load_exchange('binance', default_conf) assert isinstance(exchange, Exchange) assert isinstance(exchange, Binance) assert not isinstance(exchange, Kraken) @@ -145,7 +145,7 @@ def test_exchange_resolver(default_conf, mocker, caplog): caplog) # Test mapping - exchange = ExchangeResolver('binanceus', default_conf).exchange + exchange = ExchangeResolver.load_exchange('binanceus', default_conf) assert isinstance(exchange, Exchange) assert isinstance(exchange, Binance) assert not isinstance(exchange, Kraken) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index e7f098777..21929de2b 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -53,7 +53,8 @@ def test_load_pairlist_noexist(mocker, markets, default_conf): with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): - PairListResolver.load_pairlist('NonexistingPairList', bot.exchange, plm, default_conf, {}, 1) + PairListResolver.load_pairlist('NonexistingPairList', bot.exchange, plm, + default_conf, {}, 1) def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): From 248ef5a0eac41a82febd07ef8b1e4f421f7cce38 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 10:06:19 +0100 Subject: [PATCH 077/128] Convert HyperoptResolver to static loader --- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/resolvers/hyperopt_resolver.py | 29 +++++++++++++----------- tests/optimize/test_hyperopt.py | 6 ++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 521a4d790..a4a8f79d1 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -64,7 +64,7 @@ class Hyperopt: self.backtesting = Backtesting(self.config) - self.custom_hyperopt = HyperOptResolver(self.config).hyperopt + self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) self.custom_hyperoptloss = HyperOptLossResolver(self.config).hyperoptloss self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 05efa1164..a7f922c7b 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -20,11 +20,11 @@ class HyperOptResolver(IResolver): """ This class contains all the logic to load custom hyperopt class """ - __slots__ = ['hyperopt'] - def __init__(self, config: Dict) -> None: + @staticmethod + def load_hyperopt(config: Dict) -> IHyperOpt: """ - Load the custom class from config parameter + Load the custom hyperopt class from config parameter :param config: configuration dictionary """ if not config.get('hyperopt'): @@ -33,21 +33,23 @@ class HyperOptResolver(IResolver): hyperopt_name = config['hyperopt'] - self.hyperopt = self._load_hyperopt(hyperopt_name, config, - extra_dir=config.get('hyperopt_path')) + hyperopt = HyperOptResolver._load_hyperopt(hyperopt_name, config, + extra_dir=config.get('hyperopt_path')) - if not hasattr(self.hyperopt, 'populate_indicators'): + if not hasattr(hyperopt, 'populate_indicators'): logger.warning("Hyperopt class does not provide populate_indicators() method. " "Using populate_indicators from the strategy.") - if not hasattr(self.hyperopt, 'populate_buy_trend'): + if not hasattr(hyperopt, 'populate_buy_trend'): logger.warning("Hyperopt class does not provide populate_buy_trend() method. " "Using populate_buy_trend from the strategy.") - if not hasattr(self.hyperopt, 'populate_sell_trend'): + if not hasattr(hyperopt, 'populate_sell_trend'): logger.warning("Hyperopt class does not provide populate_sell_trend() method. " "Using populate_sell_trend from the strategy.") + return hyperopt + @staticmethod def _load_hyperopt( - self, hyperopt_name: str, config: Dict, extra_dir: Optional[str] = None) -> IHyperOpt: + hyperopt_name: str, config: Dict, extra_dir: Optional[str] = None) -> IHyperOpt: """ Search and loads the specified hyperopt. :param hyperopt_name: name of the module to import @@ -57,11 +59,12 @@ class HyperOptResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, extra_dir=extra_dir) + abs_paths = IResolver.build_search_paths(config, current_path=current_path, + user_subdir=USERPATH_HYPEROPTS, + extra_dir=extra_dir) - hyperopt = self._load_object(paths=abs_paths, object_type=IHyperOpt, - object_name=hyperopt_name, kwargs={'config': config}) + hyperopt = IResolver._load_object(paths=abs_paths, object_type=IHyperOpt, + object_name=hyperopt_name, kwargs={'config': config}) if hyperopt: return hyperopt raise OperationalException( diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 29b8b5b16..37de32ab0 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -163,7 +163,7 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: MagicMock(return_value=hyperopt(default_conf)) ) default_conf.update({'hyperopt': 'DefaultHyperOpt'}) - x = HyperOptResolver(default_conf).hyperopt + x = HyperOptResolver.load_hyperopt(default_conf) assert not hasattr(x, 'populate_indicators') assert not hasattr(x, 'populate_buy_trend') assert not hasattr(x, 'populate_sell_trend') @@ -180,7 +180,7 @@ def test_hyperoptresolver_wrongname(mocker, default_conf, caplog) -> None: default_conf.update({'hyperopt': "NonExistingHyperoptClass"}) with pytest.raises(OperationalException, match=r'Impossible to load Hyperopt.*'): - HyperOptResolver(default_conf).hyperopt + HyperOptResolver.load_hyperopt(default_conf) def test_hyperoptresolver_noname(default_conf): @@ -188,7 +188,7 @@ def test_hyperoptresolver_noname(default_conf): with pytest.raises(OperationalException, match="No Hyperopt set. Please use `--hyperopt` to specify " "the Hyperopt class to use."): - HyperOptResolver(default_conf) + HyperOptResolver.load_hyperopt(default_conf) def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None: From 6d5aca4f323dd06cac94cbf598173be6fb3ed645 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 10:09:08 +0100 Subject: [PATCH 078/128] Convert hyperoptloss resolver to static loader --- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/resolvers/hyperopt_resolver.py | 26 +++++++++++++----------- tests/optimize/test_hyperopt.py | 14 ++++++------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a4a8f79d1..48f883ac5 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -66,7 +66,7 @@ class Hyperopt: self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) - self.custom_hyperoptloss = HyperOptLossResolver(self.config).hyperoptloss + self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function self.trials_file = (self.config['user_data_dir'] / diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index a7f922c7b..0726b0627 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -77,9 +77,9 @@ class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class """ - __slots__ = ['hyperoptloss'] - def __init__(self, config: Dict) -> None: + @staticmethod + def load_hyperoptloss(config: Dict) -> IHyperOptLoss: """ Load the custom class from config parameter :param config: configuration dictionary @@ -89,20 +89,21 @@ class HyperOptLossResolver(IResolver): # default hyperopt loss hyperoptloss_name = config.get('hyperopt_loss') or DEFAULT_HYPEROPT_LOSS - self.hyperoptloss = self._load_hyperoptloss( + hyperoptloss = HyperOptLossResolver._load_hyperoptloss( hyperoptloss_name, config, extra_dir=config.get('hyperopt_path')) # Assign ticker_interval to be used in hyperopt - self.hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) + hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) - if not hasattr(self.hyperoptloss, 'hyperopt_loss_function'): + if not hasattr(hyperoptloss, 'hyperopt_loss_function'): raise OperationalException( f"Found HyperoptLoss class {hyperoptloss_name} does not " "implement `hyperopt_loss_function`.") + return hyperoptloss - def _load_hyperoptloss( - self, hyper_loss_name: str, config: Dict, - extra_dir: Optional[str] = None) -> IHyperOptLoss: + @staticmethod + def _load_hyperoptloss(hyper_loss_name: str, config: Dict, + extra_dir: Optional[str] = None) -> IHyperOptLoss: """ Search and loads the specified hyperopt loss class. :param hyper_loss_name: name of the module to import @@ -112,11 +113,12 @@ class HyperOptLossResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, extra_dir=extra_dir) + abs_paths = IResolver.build_search_paths(config, current_path=current_path, + user_subdir=USERPATH_HYPEROPTS, + extra_dir=extra_dir) - hyperoptloss = self._load_object(paths=abs_paths, object_type=IHyperOptLoss, - object_name=hyper_loss_name) + hyperoptloss = IResolver._load_object(paths=abs_paths, object_type=IHyperOptLoss, + object_name=hyper_loss_name) if hyperoptloss: return hyperoptloss diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 37de32ab0..9c6e73c53 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -198,7 +198,7 @@ def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None: 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver._load_hyperoptloss', MagicMock(return_value=hl) ) - x = HyperOptLossResolver(default_conf).hyperoptloss + x = HyperOptLossResolver.load_hyperoptloss(default_conf) assert hasattr(x, "hyperopt_loss_function") @@ -206,7 +206,7 @@ def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None: default_conf.update({'hyperopt_loss': "NonExistingLossClass"}) with pytest.raises(OperationalException, match=r'Impossible to load HyperoptLoss.*'): - HyperOptLossResolver(default_conf).hyperopt + HyperOptLossResolver.load_hyperoptloss(default_conf) def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None: @@ -286,7 +286,7 @@ def test_start_filelock(mocker, default_conf, caplog) -> None: def test_loss_calculation_prefer_correct_trade_count(default_conf, hyperopt_results) -> None: - hl = HyperOptLossResolver(default_conf).hyperoptloss + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function(hyperopt_results, 600) over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100) under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100) @@ -298,7 +298,7 @@ def test_loss_calculation_prefer_shorter_trades(default_conf, hyperopt_results) resultsb = hyperopt_results.copy() resultsb.loc[1, 'trade_duration'] = 20 - hl = HyperOptLossResolver(default_conf).hyperoptloss + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) longer = hl.hyperopt_loss_function(hyperopt_results, 100) shorter = hl.hyperopt_loss_function(resultsb, 100) assert shorter < longer @@ -310,7 +310,7 @@ def test_loss_calculation_has_limited_profit(default_conf, hyperopt_results) -> results_under = hyperopt_results.copy() results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - hl = HyperOptLossResolver(default_conf).hyperoptloss + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function(hyperopt_results, 600) over = hl.hyperopt_loss_function(results_over, 600) under = hl.hyperopt_loss_function(results_under, 600) @@ -325,7 +325,7 @@ def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> N results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 default_conf.update({'hyperopt_loss': 'SharpeHyperOptLoss'}) - hl = HyperOptLossResolver(default_conf).hyperoptloss + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), datetime(2019, 1, 1), datetime(2019, 5, 1)) over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), @@ -343,7 +343,7 @@ def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) - hl = HyperOptLossResolver(default_conf).hyperoptloss + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), datetime(2019, 1, 1), datetime(2019, 5, 1)) over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), From c6d22339788e9686552d171bf19cb00d41f7cb0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 10:23:48 +0100 Subject: [PATCH 079/128] Convert StrategyLoader to static loader --- docs/strategy_analysis_example.md | 6 +- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 4 +- freqtrade/optimize/edge_cli.py | 2 +- freqtrade/plot/plotting.py | 2 +- freqtrade/resolvers/strategy_resolver.py | 65 ++++---- .../templates/strategy_analysis_example.ipynb | 6 +- tests/strategy/test_strategy.py | 141 +++++++++--------- 8 files changed, 116 insertions(+), 112 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 9e61bda65..cc6b9805f 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -44,9 +44,9 @@ candles.head() ```python # Load strategy using values set above from freqtrade.resolvers import StrategyResolver -strategy = StrategyResolver({'strategy': strategy_name, - 'user_data_dir': user_data_dir, - 'strategy_path': strategy_location}).strategy +strategy = StrategyResolver.load_strategy({'strategy': strategy_name, + 'user_data_dir': user_data_dir, + 'strategy_path': strategy_location}) # Generate buy/sell signals using strategy df = strategy.analyze_ticker(candles, {'pair': pair}) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 001b7f02d..1b89cc75b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -55,7 +55,7 @@ class FreqtradeBot: self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) - self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options validate_config_consistency(config) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index bab997cb1..ffa112bd5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -75,12 +75,12 @@ class Backtesting: for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat - self.strategylist.append(StrategyResolver(stratconf).strategy) + self.strategylist.append(StrategyResolver.load_strategy(stratconf)) validate_config_consistency(stratconf) else: # No strategy list specified, only one strategy - self.strategylist.append(StrategyResolver(self.config).strategy) + self.strategylist.append(StrategyResolver.load_strategy(self.config)) validate_config_consistency(self.config) if "ticker_interval" not in self.config: diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 3848623db..ea5cc663d 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -34,7 +34,7 @@ class EdgeCli: remove_credentials(self.config) self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT self.exchange = Exchange(self.config) - self.strategy = StrategyResolver(self.config).strategy + self.strategy = StrategyResolver.load_strategy(self.config) validate_config_consistency(self.config) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 85089af9c..7cd4ab854 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -340,7 +340,7 @@ def load_and_plot_trades(config: Dict[str, Any]): - Generate plot files :return: None """ - strategy = StrategyResolver(config).strategy + strategy = StrategyResolver.load_strategy(config) plot_elements = init_plotscript(config) trades = plot_elements['trades'] diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index a2d14fbf3..6d3fe5ff9 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -20,12 +20,11 @@ logger = logging.getLogger(__name__) class StrategyResolver(IResolver): """ - This class contains all the logic to load custom strategy class + This class contains the logic to load custom strategy class """ - __slots__ = ['strategy'] - - def __init__(self, config: Optional[Dict] = None) -> None: + @staticmethod + def load_strategy(config: Optional[Dict] = None) -> IStrategy: """ Load the custom class from config parameter :param config: configuration dictionary or None @@ -37,9 +36,9 @@ class StrategyResolver(IResolver): "the strategy class to use.") strategy_name = config['strategy'] - self.strategy: IStrategy = self._load_strategy(strategy_name, - config=config, - extra_dir=config.get('strategy_path')) + strategy: IStrategy = StrategyResolver._load_strategy( + strategy_name, config=config, + extra_dir=config.get('strategy_path')) # make sure ask_strategy dict is available if 'ask_strategy' not in config: @@ -68,9 +67,11 @@ class StrategyResolver(IResolver): ] for attribute, default, ask_strategy in attributes: if ask_strategy: - self._override_attribute_helper(config['ask_strategy'], attribute, default) + StrategyResolver._override_attribute_helper(strategy, config['ask_strategy'], + attribute, default) else: - self._override_attribute_helper(config, attribute, default) + StrategyResolver._override_attribute_helper(strategy, config, + attribute, default) # Loop this list again to have output combined for attribute, _, exp in attributes: @@ -80,14 +81,16 @@ class StrategyResolver(IResolver): logger.info("Strategy using %s: %s", attribute, config[attribute]) # Sort and apply type conversions - self.strategy.minimal_roi = OrderedDict(sorted( - {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), + strategy.minimal_roi = OrderedDict(sorted( + {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), key=lambda t: t[0])) - self.strategy.stoploss = float(self.strategy.stoploss) + strategy.stoploss = float(strategy.stoploss) - self._strategy_sanity_validations() + StrategyResolver._strategy_sanity_validations(strategy) + return strategy - def _override_attribute_helper(self, config, attribute: str, default): + @staticmethod + def _override_attribute_helper(strategy, config, attribute: str, default): """ Override attributes in the strategy. Prevalence: @@ -96,30 +99,32 @@ class StrategyResolver(IResolver): - default (if not None) """ if attribute in config: - setattr(self.strategy, attribute, config[attribute]) + setattr(strategy, attribute, config[attribute]) logger.info("Override strategy '%s' with value in config file: %s.", attribute, config[attribute]) - elif hasattr(self.strategy, attribute): - val = getattr(self.strategy, attribute) + elif hasattr(strategy, attribute): + val = getattr(strategy, attribute) # None's cannot exist in the config, so do not copy them if val is not None: config[attribute] = val # Explicitly check for None here as other "falsy" values are possible elif default is not None: - setattr(self.strategy, attribute, default) + setattr(strategy, attribute, default) config[attribute] = default - def _strategy_sanity_validations(self): - if not all(k in self.strategy.order_types for k in constants.REQUIRED_ORDERTYPES): - raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + @staticmethod + def _strategy_sanity_validations(strategy): + if not all(k in strategy.order_types for k in constants.REQUIRED_ORDERTYPES): + raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-types mapping is incomplete.") - if not all(k in self.strategy.order_time_in_force for k in constants.REQUIRED_ORDERTIF): - raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + if not all(k in strategy.order_time_in_force for k in constants.REQUIRED_ORDERTIF): + raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-time-in-force mapping is incomplete.") - def _load_strategy( - self, strategy_name: str, config: dict, extra_dir: Optional[str] = None) -> IStrategy: + @staticmethod + def _load_strategy(strategy_name: str, + config: dict, extra_dir: Optional[str] = None) -> IStrategy: """ Search and loads the specified strategy. :param strategy_name: name of the module to import @@ -129,9 +134,9 @@ class StrategyResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=constants.USERPATH_STRATEGY, - extra_dir=extra_dir) + abs_paths = IResolver.build_search_paths(config, current_path=current_path, + user_subdir=constants.USERPATH_STRATEGY, + extra_dir=extra_dir) if ":" in strategy_name: logger.info("loading base64 encoded strategy") @@ -149,8 +154,8 @@ class StrategyResolver(IResolver): # register temp path with the bot abs_paths.insert(0, temp.resolve()) - strategy = self._load_object(paths=abs_paths, object_type=IStrategy, - object_name=strategy_name, kwargs={'config': config}) + strategy = IResolver._load_object(paths=abs_paths, object_type=IStrategy, + object_name=strategy_name, kwargs={'config': config}) if strategy: strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 2876ea938..eea8fb85f 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -73,9 +73,9 @@ "source": [ "# Load strategy using values set above\n", "from freqtrade.resolvers import StrategyResolver\n", - "strategy = StrategyResolver({'strategy': strategy_name,\n", - " 'user_data_dir': user_data_dir,\n", - " 'strategy_path': strategy_location}).strategy\n", + "strategy = StrategyResolver.load_strategy({'strategy': strategy_name,\n", + " 'user_data_dir': user_data_dir,\n", + " 'strategy_path': strategy_location})\n", "\n", "# Generate buy/sell signals using strategy\n", "df = strategy.analyze_ticker(candles, {'pair': pair})\n", diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 963d36c76..116eec56b 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -39,8 +39,8 @@ def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', 'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates') }) - resolver = StrategyResolver(default_conf) - assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + strategy = StrategyResolver.load_strategy(default_conf) + assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) def test_load_strategy_base64(result, caplog, default_conf): @@ -48,8 +48,8 @@ def test_load_strategy_base64(result, caplog, default_conf): encoded_string = urlsafe_b64encode(file.read()).decode("utf-8") default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)}) - resolver = StrategyResolver(default_conf) - assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + strategy = StrategyResolver.load_strategy(default_conf) + assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) # Make sure strategy was loaded from base64 (using temp directory)!! assert log_has_re(r"Using resolved strategy SampleStrategy from '" r".*(/|\\).*(/|\\)SampleStrategy\.py'\.\.\.", caplog) @@ -57,13 +57,12 @@ def test_load_strategy_base64(result, caplog, default_conf): def test_load_strategy_invalid_directory(result, caplog, default_conf): default_conf['strategy'] = 'DefaultStrategy' - resolver = StrategyResolver(default_conf) extra_dir = Path.cwd() / 'some/path' - resolver._load_strategy('DefaultStrategy', config=default_conf, extra_dir=extra_dir) + strategy = StrategyResolver._load_strategy('DefaultStrategy', config=default_conf, extra_dir=extra_dir) assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog) - assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) def test_load_not_found_strategy(default_conf): @@ -71,7 +70,7 @@ def test_load_not_found_strategy(default_conf): with pytest.raises(OperationalException, match=r"Impossible to load Strategy 'NotFoundStrategy'. " r"This class does not exist or contains Python code errors."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_load_strategy_noname(default_conf): @@ -79,30 +78,30 @@ def test_load_strategy_noname(default_conf): with pytest.raises(OperationalException, match="No strategy set. Please use `--strategy` to specify " "the strategy class to use."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_strategy(result, default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} - assert resolver.strategy.minimal_roi[0] == 0.04 + assert strategy.minimal_roi[0] == 0.04 assert default_conf["minimal_roi"]['0'] == 0.04 - assert resolver.strategy.stoploss == -0.10 + assert strategy.stoploss == -0.10 assert default_conf['stoploss'] == -0.10 - assert resolver.strategy.ticker_interval == '5m' + assert strategy.ticker_interval == '5m' assert default_conf['ticker_interval'] == '5m' - df_indicators = resolver.strategy.advise_indicators(result, metadata=metadata) + df_indicators = strategy.advise_indicators(result, metadata=metadata) assert 'adx' in df_indicators - dataframe = resolver.strategy.advise_buy(df_indicators, metadata=metadata) + dataframe = strategy.advise_buy(df_indicators, metadata=metadata) assert 'buy' in dataframe.columns - dataframe = resolver.strategy.advise_sell(df_indicators, metadata=metadata) + dataframe = strategy.advise_sell(df_indicators, metadata=metadata) assert 'sell' in dataframe.columns @@ -114,9 +113,9 @@ def test_strategy_override_minimal_roi(caplog, default_conf): "0": 0.5 } }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.minimal_roi[0] == 0.5 + assert strategy.minimal_roi[0] == 0.5 assert log_has("Override strategy 'minimal_roi' with value in config file: {'0': 0.5}.", caplog) @@ -126,9 +125,9 @@ def test_strategy_override_stoploss(caplog, default_conf): 'strategy': 'DefaultStrategy', 'stoploss': -0.5 }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.stoploss == -0.5 + assert strategy.stoploss == -0.5 assert log_has("Override strategy 'stoploss' with value in config file: -0.5.", caplog) @@ -138,10 +137,10 @@ def test_strategy_override_trailing_stop(caplog, default_conf): 'strategy': 'DefaultStrategy', 'trailing_stop': True }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.trailing_stop - assert isinstance(resolver.strategy.trailing_stop, bool) + assert strategy.trailing_stop + assert isinstance(strategy.trailing_stop, bool) assert log_has("Override strategy 'trailing_stop' with value in config file: True.", caplog) @@ -153,13 +152,13 @@ def test_strategy_override_trailing_stop_positive(caplog, default_conf): 'trailing_stop_positive_offset': -0.2 }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.trailing_stop_positive == -0.1 + assert strategy.trailing_stop_positive == -0.1 assert log_has("Override strategy 'trailing_stop_positive' with value in config file: -0.1.", caplog) - assert resolver.strategy.trailing_stop_positive_offset == -0.2 + assert strategy.trailing_stop_positive_offset == -0.2 assert log_has("Override strategy 'trailing_stop_positive' with value in config file: -0.1.", caplog) @@ -172,10 +171,10 @@ def test_strategy_override_ticker_interval(caplog, default_conf): 'ticker_interval': 60, 'stake_currency': 'ETH' }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.ticker_interval == 60 - assert resolver.strategy.stake_currency == 'ETH' + assert strategy.ticker_interval == 60 + assert strategy.stake_currency == 'ETH' assert log_has("Override strategy 'ticker_interval' with value in config file: 60.", caplog) @@ -187,9 +186,9 @@ def test_strategy_override_process_only_new_candles(caplog, default_conf): 'strategy': 'DefaultStrategy', 'process_only_new_candles': True }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.process_only_new_candles + assert strategy.process_only_new_candles assert log_has("Override strategy 'process_only_new_candles' with value in config file: True.", caplog) @@ -207,11 +206,11 @@ def test_strategy_override_order_types(caplog, default_conf): 'strategy': 'DefaultStrategy', 'order_types': order_types }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.order_types + assert strategy.order_types for method in ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']: - assert resolver.strategy.order_types[method] == order_types[method] + assert strategy.order_types[method] == order_types[method] assert log_has("Override strategy 'order_types' with value in config file:" " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'," @@ -225,7 +224,7 @@ def test_strategy_override_order_types(caplog, default_conf): with pytest.raises(ImportError, match=r"Impossible to load Strategy 'DefaultStrategy'. " r"Order-types mapping is incomplete."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_strategy_override_order_tif(caplog, default_conf): @@ -240,11 +239,11 @@ def test_strategy_override_order_tif(caplog, default_conf): 'strategy': 'DefaultStrategy', 'order_time_in_force': order_time_in_force }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.order_time_in_force + assert strategy.order_time_in_force for method in ['buy', 'sell']: - assert resolver.strategy.order_time_in_force[method] == order_time_in_force[method] + assert strategy.order_time_in_force[method] == order_time_in_force[method] assert log_has("Override strategy 'order_time_in_force' with value in config file:" " {'buy': 'fok', 'sell': 'gtc'}.", caplog) @@ -257,7 +256,7 @@ def test_strategy_override_order_tif(caplog, default_conf): with pytest.raises(ImportError, match=r"Impossible to load Strategy 'DefaultStrategy'. " r"Order-time-in-force mapping is incomplete."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_strategy_override_use_sell_signal(caplog, default_conf): @@ -265,9 +264,9 @@ def test_strategy_override_use_sell_signal(caplog, default_conf): default_conf.update({ 'strategy': 'DefaultStrategy', }) - resolver = StrategyResolver(default_conf) - assert resolver.strategy.use_sell_signal - assert isinstance(resolver.strategy.use_sell_signal, bool) + strategy = StrategyResolver.load_strategy(default_conf) + assert strategy.use_sell_signal + assert isinstance(strategy.use_sell_signal, bool) # must be inserted to configuration assert 'use_sell_signal' in default_conf['ask_strategy'] assert default_conf['ask_strategy']['use_sell_signal'] @@ -278,10 +277,10 @@ def test_strategy_override_use_sell_signal(caplog, default_conf): 'use_sell_signal': False, }, }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert not resolver.strategy.use_sell_signal - assert isinstance(resolver.strategy.use_sell_signal, bool) + assert not strategy.use_sell_signal + assert isinstance(strategy.use_sell_signal, bool) assert log_has("Override strategy 'use_sell_signal' with value in config file: False.", caplog) @@ -290,9 +289,9 @@ def test_strategy_override_use_sell_profit_only(caplog, default_conf): default_conf.update({ 'strategy': 'DefaultStrategy', }) - resolver = StrategyResolver(default_conf) - assert not resolver.strategy.sell_profit_only - assert isinstance(resolver.strategy.sell_profit_only, bool) + strategy = StrategyResolver.load_strategy(default_conf) + assert not strategy.sell_profit_only + assert isinstance(strategy.sell_profit_only, bool) # must be inserted to configuration assert 'sell_profit_only' in default_conf['ask_strategy'] assert not default_conf['ask_strategy']['sell_profit_only'] @@ -303,10 +302,10 @@ def test_strategy_override_use_sell_profit_only(caplog, default_conf): 'sell_profit_only': True, }, }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.sell_profit_only - assert isinstance(resolver.strategy.sell_profit_only, bool) + assert strategy.sell_profit_only + assert isinstance(strategy.sell_profit_only, bool) assert log_has("Override strategy 'sell_profit_only' with value in config file: True.", caplog) @@ -315,11 +314,11 @@ def test_deprecate_populate_indicators(result, default_conf): default_location = path.join(path.dirname(path.realpath(__file__))) default_conf.update({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - indicators = resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + indicators = strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -328,7 +327,7 @@ def test_deprecate_populate_indicators(result, default_conf): with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.advise_buy(indicators, {'pair': 'ETH/BTC'}) + strategy.advise_buy(indicators, {'pair': 'ETH/BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -337,7 +336,7 @@ def test_deprecate_populate_indicators(result, default_conf): with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.advise_sell(indicators, {'pair': 'ETH_BTC'}) + strategy.advise_sell(indicators, {'pair': 'ETH_BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -349,47 +348,47 @@ def test_call_deprecated_function(result, monkeypatch, default_conf): default_location = path.join(path.dirname(path.realpath(__file__))) default_conf.update({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} # Make sure we are using a legacy function - assert resolver.strategy._populate_fun_len == 2 - assert resolver.strategy._buy_fun_len == 2 - assert resolver.strategy._sell_fun_len == 2 - assert resolver.strategy.INTERFACE_VERSION == 1 + assert strategy._populate_fun_len == 2 + assert strategy._buy_fun_len == 2 + assert strategy._sell_fun_len == 2 + assert strategy.INTERFACE_VERSION == 1 - indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) + indicator_df = strategy.advise_indicators(result, metadata=metadata) assert isinstance(indicator_df, DataFrame) assert 'adx' in indicator_df.columns - buydf = resolver.strategy.advise_buy(result, metadata=metadata) + buydf = strategy.advise_buy(result, metadata=metadata) assert isinstance(buydf, DataFrame) assert 'buy' in buydf.columns - selldf = resolver.strategy.advise_sell(result, metadata=metadata) + selldf = strategy.advise_sell(result, metadata=metadata) assert isinstance(selldf, DataFrame) assert 'sell' in selldf def test_strategy_interface_versioning(result, monkeypatch, default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} # Make sure we are using a legacy function - assert resolver.strategy._populate_fun_len == 3 - assert resolver.strategy._buy_fun_len == 3 - assert resolver.strategy._sell_fun_len == 3 - assert resolver.strategy.INTERFACE_VERSION == 2 + assert strategy._populate_fun_len == 3 + assert strategy._buy_fun_len == 3 + assert strategy._sell_fun_len == 3 + assert strategy.INTERFACE_VERSION == 2 - indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) + indicator_df = strategy.advise_indicators(result, metadata=metadata) assert isinstance(indicator_df, DataFrame) assert 'adx' in indicator_df.columns - buydf = resolver.strategy.advise_buy(result, metadata=metadata) + buydf = strategy.advise_buy(result, metadata=metadata) assert isinstance(buydf, DataFrame) assert 'buy' in buydf.columns - selldf = resolver.strategy.advise_sell(result, metadata=metadata) + selldf = strategy.advise_sell(result, metadata=metadata) assert isinstance(selldf, DataFrame) assert 'sell' in selldf From 90cabd7c21306f6f7fda068e40936e7f005ca825 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 10:46:35 +0100 Subject: [PATCH 080/128] Wrap line --- tests/strategy/test_strategy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 116eec56b..ce7ac1741 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -58,7 +58,8 @@ def test_load_strategy_base64(result, caplog, default_conf): def test_load_strategy_invalid_directory(result, caplog, default_conf): default_conf['strategy'] = 'DefaultStrategy' extra_dir = Path.cwd() / 'some/path' - strategy = StrategyResolver._load_strategy('DefaultStrategy', config=default_conf, extra_dir=extra_dir) + strategy = StrategyResolver._load_strategy('DefaultStrategy', config=default_conf, + extra_dir=extra_dir) assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog) From bb8acc61db3685f09e34169c0e4f8bb17fac8588 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 15:09:17 +0100 Subject: [PATCH 081/128] Convert datadir within config to Path (it's used as Path all the time!) --- freqtrade/configuration/configuration.py | 2 +- freqtrade/configuration/directory_operations.py | 4 ++-- freqtrade/data/dataprovider.py | 3 +-- freqtrade/utils.py | 9 ++++----- tests/test_configuration.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 001e89303..f73b52c10 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -403,7 +403,7 @@ class Configuration: config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: # Fall back to /dl_path/pairs.json - pairs_file = Path(config['datadir']) / "pairs.json" + pairs_file = config['datadir'] / "pairs.json" if pairs_file.exists(): with pairs_file.open('r') as f: config['pairs'] = json_load(f) diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 3dd76a025..556f27742 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -9,7 +9,7 @@ from freqtrade.constants import USER_DATA_FILES logger = logging.getLogger(__name__) -def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str: +def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Path: folder = Path(datadir) if datadir else Path(f"{config['user_data_dir']}/data") if not datadir: @@ -20,7 +20,7 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str if not folder.is_dir(): folder.mkdir(parents=True) logger.info(f'Created data directory: {datadir}') - return str(folder) + return folder def create_userdata_dir(directory: str, create_dir=False) -> Path: diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 7b7159145..2964d1cb7 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -5,7 +5,6 @@ including Klines, tickers, historic data Common Interface for bot and strategy to access data. """ import logging -from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame @@ -65,7 +64,7 @@ class DataProvider: """ return load_pair_history(pair=pair, timeframe=timeframe or self._config['ticker_interval'], - datadir=Path(self._config['datadir']) + datadir=self._config['datadir'] ) def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame: diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 9e01c7ea6..2ae9e1ecb 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -191,9 +191,8 @@ def start_download_data(args: Dict[str, Any]) -> None: "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") - dl_path = Path(config['datadir']) logger.info(f'About to download pairs: {config["pairs"]}, ' - f'intervals: {config["timeframes"]} to {dl_path}') + f'intervals: {config["timeframes"]} to {config["datadir"]}') pairs_not_available: List[str] = [] @@ -203,17 +202,17 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config["pairs"], datadir=Path(config['datadir']), + exchange, pairs=config["pairs"], datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( pairs=config["pairs"], timeframes=config["timeframes"], - datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 292d53315..6564417b3 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -977,7 +977,7 @@ def test_pairlist_resolving_fallback(mocker): assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] assert config['exchange']['name'] == 'binance' - assert config['datadir'] == str(Path.cwd() / "user_data/data/binance") + assert config['datadir'] == Path.cwd() / "user_data/data/binance" @pytest.mark.parametrize("setting", [ From ecbb77c17fa3130e2d3dbef2044a7c2d8447685b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 15:13:55 +0100 Subject: [PATCH 082/128] Add forgotten option --- freqtrade/plot/plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 85089af9c..62081194f 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -37,7 +37,7 @@ def init_plotscript(config): timerange = TimeRange.parse_timerange(config.get("timerange")) tickers = history.load_data( - datadir=Path(str(config.get("datadir"))), + datadir=config.get("datadir"), pairs=pairs, timeframe=config.get('ticker_interval', '5m'), timerange=timerange, From c6b9c8eca0722305f6d450cf7d9cada9cae216ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 19:32:31 +0100 Subject: [PATCH 083/128] Forgot to save --- freqtrade/edge/__init__.py | 4 ++-- freqtrade/optimize/backtesting.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e56071a98..e1dfe8e25 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -96,7 +96,7 @@ class Edge: if self._refresh_pairs: history.refresh_data( - datadir=Path(self.config['datadir']), + datadir=self.config['datadir'], pairs=pairs, exchange=self.exchange, timeframe=self.strategy.ticker_interval, @@ -104,7 +104,7 @@ class Edge: ) data = history.load_data( - datadir=Path(self.config['datadir']), + datadir=self.config['datadir'], pairs=pairs, timeframe=self.strategy.ticker_interval, timerange=self._timerange, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 726257cdd..630cd106f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -109,7 +109,7 @@ class Backtesting: 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( - datadir=Path(self.config['datadir']), + datadir=self.config['datadir'], pairs=self.config['exchange']['pair_whitelist'], timeframe=self.timeframe, timerange=timerange, From 0ac5e5035c499e07569f63dca45ac11938a601c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 20:43:31 +0100 Subject: [PATCH 084/128] Remove unused import --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e1dfe8e25..9ad2485ef 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,7 +1,6 @@ # pragma pylint: disable=W0603 """ Edge positioning package """ import logging -from pathlib import Path from typing import Any, Dict, NamedTuple import arrow From 690eb2a52b1883e3faff22a29a2f68dcaad4aa2f Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Tue, 24 Dec 2019 07:19:35 +0300 Subject: [PATCH 085/128] configuration.md reviewed --- docs/configuration.md | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8ae824277..b3292ee74 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -388,48 +388,51 @@ The valid values are: ## Prices used for orders -Prices for regular orders can be controlled via the parameter structures `bid_strategy` for Buying, and `ask_strategy` for selling. -Prices are always retrieved right before an order is placed, either by querying the `fetch_ticker()` endpoint of the exchange (usually `/ticker`), or by using the orderbook. +Prices for regular orders can be controlled via the parameter structures `bid_strategy` for buying and `ask_strategy` for selling. +Prices are always retrieved right before an order is placed, either by querying the exchange tickers or by using the orderbook data. + +!!! Note + Orderbook data used by Freqtrade are the data retrieved from exchange by the ccxt's function `fetch_order_book()`, i.e. are usually data from the L2-aggregated orderbook, while the ticker data are the structures returned by the ccxt's `fetch_ticker()`/`fetch_tickers()` functions. Refer to the ccxt library [documentation](https://github.com/ccxt/ccxt/wiki/Manual#market-data) for more details. ### Buy price #### Check depth of market -When enabling `bid_strategy.check_depth_of_market.enabled=True`, then buy signals will be filtered based on the orderbook size for each size (sum of all amounts). -Orderbook bid size is then divided by Orderbook ask size - and the resulting delta is compared to `bid_strategy.check_depth_of_market.bids_to_ask_delta`, and a buy is only executed if the orderbook delta is bigger or equal to the configured delta. +When check depth of market is enabled (i.e. `bid_strategy.check_depth_of_market.enabled=True`), the buy signals are filtered based on the orderbook depth (sum of all amounts) for each orderbook side. + +Orderbook `bid` (buy) side depth is then divided by the orderbook `ask` (sell) side depth and the resulting delta is compared to the value of the `bid_strategy.check_depth_of_market.bids_to_ask_delta` parameter. The buy order is only executed if the orderbook delta is greater than or equal to the configured delta value. !!! Note - A calculated delta below 1 means that sell order size is greater, while value greater than 1 means buy order size is higher + A delta value below 1 means that `ask` (sell) orderbook side depth is greater than the depth of the `bid` (buy) orderbook side, while a value greater than 1 means opposite (depth of the buy side is higher than the depth of the sell side). #### Buy price with Orderbook enabled -When buying with the orderbook enabled (`bid_strategy.use_order_book=True`) - Freqtrade will fetch the `bid_strategy.order_book_top` entries in the orderbook, and will then use the entry specified as `bid_strategy.order_book_top` on the `bids` side of the orderbook. 1 specifies the topmost entry in the Orderbook - while 2 would use the 2nd entry in the Orderbook. +When buying with the orderbook enabled (`bid_strategy.use_order_book=True`), Freqtrade fetches the `bid_strategy.order_book_top` entries from the orderbook and then uses the entry specified as `bid_strategy.order_book_top` on the `bid` (buy) side of the orderbook. 1 specifies the topmost entry in the orderbook, while 2 would use the 2nd entry in the orderbook, and so on. -#### Buy price without Orderbook +#### Buy price without Orderbook enabled -When not using orderbook (`bid_strategy.use_order_book=False`), then Freqtrade will use the best `ask` price based on a call to `fetch_ticker()` if it's below the `last` traded price. -Otherwise, it'll calculate a rate between `ask` and `last` price. +When not using orderbook (`bid_strategy.use_order_book=False`), Freqtrade uses the best `ask` (sell) price from the ticker if it's below the `last` traded price from the ticker. Otherwise (when the `ask` price is not below the `last` price), it calculates a rate between `ask` and `last` price. -The `bid_strategy.ask_last_balance` configuration parameter controls this. A value of `0.0` will use `ask` price, `1.0` will use the `last` price and values between those interpolate between ask and last -price. -Using `ask` price will guarantee quick success in bid, but bot will also end up paying more than what would have been necessary. +The `bid_strategy.ask_last_balance` configuration parameter controls this. A value of `0.0` will use `ask` price, while `1.0` will use the `last` price and values between those interpolate between ask and last price. + +Using `ask` price often guarantees quicker success in the bid, but the bot can also end up paying more than what would have been necessary. ### Sell price #### Sell price with Orderbook enabled -When selling with the Orderbook enabled (`ask_strategy.use_order_book=True`) - Freqtrade will fetch the `ask_strategy.order_book_max` entries in the orderbook. Freqtrade will then validate each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the `ask` side for a profitable sell-possibility based on the strategy configuration and will place a sell order at the first profitable spot. +When selling with the orderbook enabled (`ask_strategy.use_order_book=True`), Freqtrade fetches the `ask_strategy.order_book_max` entries in the orderbook. Then each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the `ask` orderbook side are validated for a profitable sell-possibility based on the strategy configuration and the sell order is placed at the first profitable spot. + +The idea here is to place the sell order early, to be ahead in the queue. -The idea here is to place the sell-order early, to be ahead in the queue. A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting `ask_strategy.order_book_min` and `ask_strategy.order_book_max` to the same number. !!! Warning "Orderbook and stoploss_on_exchange" - Using `ask_strategy.order_book_max` higher than 1 may increase the risk, since an eventual [stoploss on exchange](#understand-order_types) will be need to be cancelled as soon as the order is placed. + Using `ask_strategy.order_book_max` higher than 1 may increase the risk, since an eventual [stoploss on exchange](#understand-order_types) will be needed to be cancelled as soon as the order is placed. +#### Sell price without Orderbook enabled -#### Sell price without orderbook - -When not using orderbook (`ask_strategy.use_order_book=False`), then the best `bid` will be used as sell rate based on a call to `fetch_ticker()`. +When not using orderbook (`ask_strategy.use_order_book=False`), the `bid` price from the ticker will be used as the sell price. ## Pairlists From f487dac047bfb1d61a93c5f45d9a08ddfcdab5bc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 06:27:11 +0100 Subject: [PATCH 086/128] FIx bug in dry-run wallets causing balances to stay there after trades are sold --- freqtrade/wallets.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index dd706438f..54c3f9138 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -58,13 +58,15 @@ class Wallets: - Subtract currently tied up stake_amount in open trades - update balances for currencies currently in trades """ + # Recreate _wallets to reset closed trade balances + _wallets = {} closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all() open_trades = Trade.get_trades(Trade.is_open.is_(True)).all() tot_profit = sum([trade.calc_profit() for trade in closed_trades]) tot_in_trades = sum([trade.stake_amount for trade in open_trades]) current_stake = self.start_cap + tot_profit - tot_in_trades - self._wallets[self._config['stake_currency']] = Wallet( + _wallets[self._config['stake_currency']] = Wallet( self._config['stake_currency'], current_stake, 0, @@ -73,12 +75,13 @@ class Wallets: for trade in open_trades: curr = trade.pair.split('/')[0] - self._wallets[curr] = Wallet( + _wallets[curr] = Wallet( curr, trade.amount, 0, trade.amount ) + self._wallets = _wallets def _update_live(self) -> None: From 33cfeaf9b01a51fc16d3dd9a437a6ffc5c670955 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 06:31:05 +0100 Subject: [PATCH 087/128] Remove i.e. where it doesn't fit --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index b3292ee74..b1b03c721 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -398,7 +398,7 @@ Prices are always retrieved right before an order is placed, either by querying #### Check depth of market -When check depth of market is enabled (i.e. `bid_strategy.check_depth_of_market.enabled=True`), the buy signals are filtered based on the orderbook depth (sum of all amounts) for each orderbook side. +When check depth of market is enabled (`bid_strategy.check_depth_of_market.enabled=True`), the buy signals are filtered based on the orderbook depth (sum of all amounts) for each orderbook side. Orderbook `bid` (buy) side depth is then divided by the orderbook `ask` (sell) side depth and the resulting delta is compared to the value of the `bid_strategy.check_depth_of_market.bids_to_ask_delta` parameter. The buy order is only executed if the orderbook delta is greater than or equal to the configured delta value. From b8442d536a41e4b93872618f38239fadcaa20f28 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 06:47:25 +0100 Subject: [PATCH 088/128] Update integration test to also test dry-run-wallets --- tests/test_integration.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 728e96d55..c5d08ec22 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -118,12 +118,10 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc default_conf['max_open_trades'] = 5 default_conf['forcebuy_enable'] = True default_conf['stake_amount'] = 'unlimited' + default_conf['dry_run_wallet'] = 1000 default_conf['exchange']['name'] = 'binance' default_conf['telegram']['enabled'] = True mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock( - side_effect=[1000, 800, 600, 400, 200] - )) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, @@ -138,6 +136,14 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc update_trade_state=MagicMock(), _notify_sell=MagicMock(), ) + should_sell_mock = MagicMock(side_effect=[ + SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), + SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL), + SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), + SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), + SellCheckTuple(sell_flag=None, sell_type=SellType.NONE)] + ) + mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) freqtrade = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtrade) @@ -158,3 +164,20 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc for trade in trades: assert trade.stake_amount == 200 + # Reset trade open order id's + trade.open_order_id = None + trades = Trade.get_open_trades() + assert len(trades) == 5 + bals = freqtrade.wallets.get_all_balances() + + freqtrade.process_maybe_execute_sells(trades) + trades = Trade.get_open_trades() + # One trade sold + assert len(trades) == 4 + # Validate that balance of sold trade is not in dry-run balances anymore. + bals2 = freqtrade.wallets.get_all_balances() + assert bals != bals2 + assert len(bals) == 6 + assert len(bals2) == 5 + assert 'LTC' in bals + assert 'LTC' not in bals2 From a105e5664a3d166a2c641a0da8ce10fea5f55d7b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 06:58:30 +0100 Subject: [PATCH 089/128] Align /balance output to show everything in stake currency the conversation to BTC does not make sense --- freqtrade/rpc/rpc.py | 3 ++- freqtrade/rpc/telegram.py | 4 ++-- tests/rpc/test_rpc_telegram.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3b4b7570a..d6d442df5 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -341,13 +341,14 @@ class RPC: raise RPCException('All balances are zero.') symbol = fiat_display_currency - value = self._fiat_converter.convert_amount(total, 'BTC', + value = self._fiat_converter.convert_amount(total, stake_currency, symbol) if self._fiat_converter else 0 return { 'currencies': output, 'total': total, 'symbol': symbol, 'value': value, + 'stake': stake_currency, 'note': 'Simulated balances' if self._freqtrade.config.get('dry_run', False) else '' } diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e0e2afd7b..e9ecdcff6 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -335,7 +335,7 @@ class Telegram(RPC): output = '' if self._config['dry_run']: output += ( - f"*Warning:*Simulated balances in Dry Mode.\n" + f"*Warning:* Simulated balances in Dry Mode.\n" "This mode is still experimental!\n" "Starting capital: " f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" @@ -358,7 +358,7 @@ class Telegram(RPC): output += curr_output output += "\n*Estimated Value*:\n" \ - "\t`BTC: {total: .8f}`\n" \ + "\t`{stake}: {total: .8f}`\n" \ "\t`{symbol}: {value: .2f}`\n".format(**result) self._send_msg(output) except RPCException as e: diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index b02f11394..8126ab64c 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -534,7 +534,7 @@ def test_balance_handle_empty_response_dry(default_conf, update, mocker) -> None telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 - assert "*Warning:*Simulated balances in Dry Mode." in result + assert "*Warning:* Simulated balances in Dry Mode." in result assert "Starting capital: `1000` BTC" in result From 83ed0b38c1d2fdd4305ad0c7d332ef944b24ebea Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 07:10:57 +0100 Subject: [PATCH 090/128] Wordwrap before keep it secret --- docs/configuration.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 43964e882..2f61343db 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -38,8 +38,8 @@ The prevelance for all Options is as follows: Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. -| Command | Description | -|----------|-------------| +| Parameter | Description | +|------------|-------------| | `max_open_trades` | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades).
***Datatype:*** *Positive integer or -1.* | `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#understand-stake_amount). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Positive float or `"unlimited"`.* @@ -72,9 +72,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
***Datatype:*** *String* | `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
***Datatype:*** *Boolean* -| `exchange.key` | API key to use for the exchange. Only required when you are in production mode. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* | `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)).
***Datatype:*** *List* | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)).
***Datatype:*** *List* | `exchange.ccxt_config` | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
***Datatype:*** *Dict* @@ -84,8 +84,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
***Datatype:*** *Boolean* | `pairlists` | Define one or more pairlists to be used. [More information below](#dynamic-pairlists).
*Defaults to `StaticPairList`.*
***Datatype:*** *List of Dicts* | `telegram.enabled` | Enable the usage of Telegram.
***Datatype:*** *Boolean* -| `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* | `webhook.enabled` | Enable usage of Webhook notifications
***Datatype:*** *Boolean* | `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* | `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* @@ -93,9 +93,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* | `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Boolean* | `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *IPv4* -| `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Integer between 1024 and 65535* -| `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Integer between 1024 and 65535* +| `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* | `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
***Datatype:*** *String, SQLAlchemy connect string* | `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
***Datatype:*** *Enum, either `stopped` or `running`* | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
***Datatype:*** *Boolean* From 48935d2932ca5b3317463bf78f3dda397eaf5126 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 07:25:18 +0100 Subject: [PATCH 091/128] Align edge documentation to configuration page --- docs/edge.md | 153 ++++++++++++++++++--------------------------------- 1 file changed, 54 insertions(+), 99 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index c7b088476..769e48927 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,4 +1,4 @@ -# Edge positioning +# Edge positioning This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. @@ -9,6 +9,7 @@ This page explains how to use Edge Positioning module in your bot in order to en Edge does not consider anything else than buy/sell/stoploss signals. So trailing stoploss, ROI, and everything else are ignored in its calculation. ## Introduction + Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose. But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: you give me 10$. Is it an interesting game? No, it's quite boring, isn't it? @@ -22,43 +23,61 @@ Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the The question is: How do you calculate that? How do you know if you wanna play? The answer comes to two factors: + - Win Rate - Risk Reward Ratio ### Win Rate + Win Rate (*W*) is is the mean over some amount of trades (*N*) what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only if you won or not). - W = (Number of winning trades) / (Total number of trades) = (Number of winning trades) / N +``` +W = (Number of winning trades) / (Total number of trades) = (Number of winning trades) / N +``` Complementary Loss Rate (*L*) is defined as - L = (Number of losing trades) / (Total number of trades) = (Number of losing trades) / N +``` +L = (Number of losing trades) / (Total number of trades) = (Number of losing trades) / N +``` or, which is the same, as - L = 1 – W +``` +L = 1 – W +``` ### Risk Reward Ratio + Risk Reward Ratio (*R*) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: - R = Profit / Loss +``` +R = Profit / Loss +``` Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: - Average profit = (Sum of profits) / (Number of winning trades) +``` +Average profit = (Sum of profits) / (Number of winning trades) - Average loss = (Sum of losses) / (Number of losing trades) +Average loss = (Sum of losses) / (Number of losing trades) - R = (Average profit) / (Average loss) +R = (Average profit) / (Average loss) +``` ### Expectancy + At this point we can combine *W* and *R* to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades and subtracting the percentage of losing trades, which is calculated as follows: - Expectancy Ratio = (Risk Reward Ratio X Win Rate) – Loss Rate = (R X W) – L +``` +Expectancy Ratio = (Risk Reward Ratio X Win Rate) – Loss Rate = (R X W) – L +``` So lets say your Win rate is 28% and your Risk Reward Ratio is 5: - Expectancy = (5 X 0.28) – 0.72 = 0.68 +``` +Expectancy = (5 X 0.28) – 0.72 = 0.68 +``` Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your loses. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. @@ -69,6 +88,7 @@ You can also use this value to evaluate the effectiveness of modifications to th **NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. ## How does it work? + If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example: | Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | @@ -83,6 +103,7 @@ The goal here is to find the best stoploss for the strategy in order to have the Edge module then forces stoploss value it evaluated to your strategy dynamically. ### Position size + Edge also dictates the stake amount for each trade to the bot according to the following factors: - Allowed capital at risk @@ -90,13 +111,17 @@ Edge also dictates the stake amount for each trade to the bot according to the f Allowed capital at risk is calculated as follows: - Allowed capital at risk = (Capital available_percentage) X (Allowed risk per trade) +``` +Allowed capital at risk = (Capital available_percentage) X (Allowed risk per trade) +``` Stoploss is calculated as described above against historical data. Your position size then will be: - Position size = (Allowed capital at risk) / Stoploss +``` +Position size = (Allowed capital at risk) / Stoploss +``` Example: @@ -115,100 +140,30 @@ Available capital doesn’t change before a position is sold. Let’s assume tha So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be **0.055 / 0.02 = 2.75 ETH**. ## Configurations + Edge module has following configuration options: -#### enabled -If true, then Edge will run periodically. - -(defaults to false) - -#### process_throttle_secs -How often should Edge run in seconds? - -(defaults to 3600 so one hour) - -#### calculate_since_number_of_days -Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy -Note that it downloads historical data so increasing this number would lead to slowing down the bot. - -(defaults to 7) - -#### capital_available_percentage -This is the percentage of the total capital on exchange in stake currency. - -As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital. - -(defaults to 0.5) - -#### allowed_risk -Percentage of allowed risk per trade. - -(defaults to 0.01 so 1%) - -#### stoploss_range_min - -Minimum stoploss. - -(defaults to -0.01) - -#### stoploss_range_max - -Maximum stoploss. - -(defaults to -0.10) - -#### stoploss_range_step - -As an example if this is set to -0.01 then Edge will test the strategy for \[-0.01, -0,02, -0,03 ..., -0.09, -0.10\] ranges. -Note than having a smaller step means having a bigger range which could lead to slow calculation. - -If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10. - -(defaults to -0.01) - -#### minimum_winrate - -It filters out pairs which don't have at least minimum_winrate. - -This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio. - -(defaults to 0.60) - -#### minimum_expectancy - -It filters out pairs which have the expectancy lower than this number. - -Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. - -(defaults to 0.20) - -#### min_trade_number - -When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. - -Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something. - -(defaults to 10, it is highly recommended not to decrease this number) - -#### max_trade_duration_minute - -Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign. - -**NOTICE:** While configuring this value, you should take into consideration your ticker interval. As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.). - -(defaults to 1 day, i.e. to 60 * 24 = 1440 minutes) - -#### remove_pumps - -Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off. - -(defaults to false) +| Parameter | Description | +|------------|-------------| +| `enabled` | If true, then Edge will run periodically.
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `process_throttle_secs` | How often should Edge run in seconds.
*Defaults to `3600` (once per hour).*
***Datatype:*** *Integer* +| `calculate_since_number_of_days` | Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy.
**Note** that it downloads historical data so increasing this number would lead to slowing down the bot.
*Defaults to `7` (once per hour).*
***Datatype:*** *Integer* +| `capital_available_percentage` | This is the percentage of the total capital on exchange in stake currency.
As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
*Defaults to `0.5` (once per hour).*
***Datatype:*** *Float* +| `allowed_risk` | Ratio of allowed risk per trade.
*Defaults to `0.01` (1%)).*
***Datatype:*** *Float* +| `stoploss_range_min` | Minimum stoploss.
*Defaults to `-0.01`.*
***Datatype:*** *Float* +| `stoploss_range_max` | Maximum stoploss.
*Defaults to `-0.10`.*
***Datatype:*** *Float* +| `stoploss_range_step` | As an example if this is set to -0.01 then Edge will test the strategy for `[-0.01, -0,02, -0,03 ..., -0.09, -0.10]` ranges.
**Note** than having a smaller step means having a bigger range which could lead to slow calculation.
If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
*Defaults to `-0.001`.*
***Datatype:*** *Float* +| `minimum_winrate` | It filters out pairs which don't have at least minimum_winrate.
This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio.
*Defaults to `0.60`.*
***Datatype:*** *Float* +| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
*Defaults to `0.20`.*
***Datatype:*** *Float* +| `min_trade_number` | When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable.
Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
*Defaults to `10` (it is highly recommended not to decrease this number).*
***Datatype:*** *Integer* +| `max_trade_duration_minute` | Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
**NOTICE:** While configuring this value, you should take into consideration your ticker interval. As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.).
*Defaults to `1440` (one day).*
***Datatype:*** *Integer* +| `remove_pumps` | Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
*Defaults to `false`.*
***Datatype:*** *Boolean* ## Running Edge independently You can run Edge independently in order to see in details the result. Here is an example: -```bash +``` bash freqtrade edge ``` From a68445692bfdde1d9407a8d3bc1f82d785cd3def Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 16:08:33 +0100 Subject: [PATCH 092/128] Add first steps for list-strategies --- freqtrade/configuration/arguments.py | 19 ++++++++++++++++--- freqtrade/utils.py | 8 ++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 41c5c3957..5f7bc74f1 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,6 +30,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] +ARGS_LIST_STRATEGIES = ["strategy_path"] + ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] @@ -62,7 +64,8 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperopt_show_no_header"] NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs", - "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit"] + "list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe", + "plot-profit"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] @@ -131,8 +134,9 @@ class Arguments: from freqtrade.utils import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, - start_new_hyperopt, start_new_strategy, - start_list_timeframes, start_test_pairlist, start_trading) + start_list_strategies, start_new_hyperopt, + start_new_strategy, start_list_timeframes, + start_test_pairlist, start_trading) from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit subparsers = self.parser.add_subparsers(dest='command', @@ -185,6 +189,15 @@ class Arguments: build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) + # Add list-strategies subcommand + list_strategies_cmd = subparsers.add_parser( + 'list-strategies', + help='Print available strategies.', + parents=[_common_parser], + ) + list_strategies_cmd.set_defaults(func=start_list_strategies) + self._build_args(optionlist=ARGS_LIST_STRATEGIES, parser=list_strategies_cmd) + # Add list-exchanges subcommand list_exchanges_cmd = subparsers.add_parser( 'list-exchanges', diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 18966c574..c5fc47a74 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -224,6 +224,14 @@ def start_download_data(args: Dict[str, Any]) -> None: f"on exchange {exchange.name}.") +def start_list_strategies(args: Dict[str, Any]) -> None: + """ + Print Strategies available in a folder + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + print(config) + + def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print ticker intervals (timeframes) available on Exchange From eb1040ddb7445a49168792871ab417360bdfa4a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 13:34:37 +0100 Subject: [PATCH 093/128] Convert resolvers to classmethods --- freqtrade/resolvers/exchange_resolver.py | 1 + freqtrade/resolvers/hyperopt_resolver.py | 23 +++++++++------- freqtrade/resolvers/iresolver.py | 34 +++++++++++++----------- freqtrade/resolvers/pairlist_resolver.py | 6 +++-- freqtrade/resolvers/strategy_resolver.py | 12 +++++---- tests/strategy/test_strategy.py | 2 -- 6 files changed, 43 insertions(+), 35 deletions(-) diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index e28a5cf80..2b6a731a9 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -14,6 +14,7 @@ class ExchangeResolver(IResolver): """ This class contains all the logic to load a custom exchange class """ + object_type = Exchange @staticmethod def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 0726b0627..b9c750251 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -20,6 +20,7 @@ class HyperOptResolver(IResolver): """ This class contains all the logic to load custom hyperopt class """ + object_type = IHyperOpt @staticmethod def load_hyperopt(config: Dict) -> IHyperOpt: @@ -59,12 +60,13 @@ class HyperOptResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) + abs_paths = HyperOptResolver.build_search_paths(config, current_path=current_path, + user_subdir=USERPATH_HYPEROPTS, + extra_dir=extra_dir) - hyperopt = IResolver._load_object(paths=abs_paths, object_type=IHyperOpt, - object_name=hyperopt_name, kwargs={'config': config}) + hyperopt = HyperOptResolver._load_object(paths=abs_paths, + object_name=hyperopt_name, + kwargs={'config': config}) if hyperopt: return hyperopt raise OperationalException( @@ -77,6 +79,7 @@ class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class """ + object_type = IHyperOptLoss @staticmethod def load_hyperoptloss(config: Dict) -> IHyperOptLoss: @@ -113,12 +116,12 @@ class HyperOptLossResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) + abs_paths = HyperOptLossResolver.build_search_paths(config, current_path=current_path, + user_subdir=USERPATH_HYPEROPTS, + extra_dir=extra_dir) - hyperoptloss = IResolver._load_object(paths=abs_paths, object_type=IHyperOptLoss, - object_name=hyper_loss_name) + hyperoptloss = HyperOptLossResolver._load_object(paths=abs_paths, + object_name=hyper_loss_name) if hyperoptloss: return hyperoptloss diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 0b986debb..11937c1da 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -7,7 +7,7 @@ import importlib.util import inspect import logging from pathlib import Path -from typing import Any, List, Optional, Tuple, Union, Generator +from typing import Any, Generator, List, Optional, Tuple, Type, Union logger = logging.getLogger(__name__) @@ -16,6 +16,8 @@ class IResolver: """ This class contains all the logic to load custom classes """ + # Childclasses need to override this + object_type: Type[Any] @staticmethod def build_search_paths(config, current_path: Path, user_subdir: Optional[str] = None, @@ -32,12 +34,11 @@ class IResolver: return abs_paths - @staticmethod - def _get_valid_object(object_type, module_path: Path, + @classmethod + def _get_valid_object(cls, module_path: Path, object_name: str) -> Generator[Any, None, None]: """ Generator returning objects with matching object_type and object_name in the path given. - :param object_type: object_type (class) :param module_path: absolute path to the module :param object_name: Class name of the object :return: generator containing matching objects @@ -55,19 +56,21 @@ class IResolver: valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) - if object_name == name and object_type in obj.__bases__ + if object_name == name and cls.object_type in obj.__bases__ ) return valid_objects_gen - @staticmethod - def _search_object(directory: Path, object_type, object_name: str, + @classmethod + def _search_object(cls, directory: Path, object_name: str, kwargs: dict = {}) -> Union[Tuple[Any, Path], Tuple[None, None]]: """ Search for the objectname in the given directory :param directory: relative or absolute directory path + :param object_name: ClassName of the object to load :return: object instance """ - logger.debug("Searching for %s %s in '%s'", object_type.__name__, object_name, directory) + logger.debug("Searching for %s %s in '%s'", + cls.object_type.__name__, object_name, directory) for entry in directory.iterdir(): # Only consider python files if not str(entry).endswith('.py'): @@ -75,14 +78,14 @@ class IResolver: continue module_path = entry.resolve() - obj = next(IResolver._get_valid_object(object_type, module_path, object_name), None) + obj = next(cls._get_valid_object(module_path, object_name), None) if obj: return (obj(**kwargs), module_path) return (None, None) - @staticmethod - def _load_object(paths: List[Path], object_type, object_name: str, + @classmethod + def _load_object(cls, paths: List[Path], object_name: str, kwargs: dict = {}) -> Optional[Any]: """ Try to load object from path list. @@ -90,13 +93,12 @@ class IResolver: for _path in paths: try: - (module, module_path) = IResolver._search_object(directory=_path, - object_type=object_type, - object_name=object_name, - kwargs=kwargs) + (module, module_path) = cls._search_object(directory=_path, + object_name=object_name, + kwargs=kwargs) if module: logger.info( - f"Using resolved {object_type.__name__.lower()[1:]} {object_name} " + f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} " f"from '{module_path}'...") return module except FileNotFoundError: diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 611660ff4..00ebc03aa 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -17,6 +17,7 @@ class PairListResolver(IResolver): """ This class contains all the logic to load custom PairList class """ + object_type = IPairList @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, @@ -53,8 +54,9 @@ class PairListResolver(IResolver): abs_paths = IResolver.build_search_paths(config, current_path=current_path, user_subdir=None, extra_dir=None) - pairlist = IResolver._load_object(paths=abs_paths, object_type=IPairList, - object_name=pairlist_name, kwargs=kwargs) + pairlist = PairListResolver._load_object(paths=abs_paths, + object_name=pairlist_name, + kwargs=kwargs) if pairlist: return pairlist raise OperationalException( diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 6d3fe5ff9..654103377 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -22,6 +22,7 @@ class StrategyResolver(IResolver): """ This class contains the logic to load custom strategy class """ + object_type = IStrategy @staticmethod def load_strategy(config: Optional[Dict] = None) -> IStrategy: @@ -134,9 +135,9 @@ class StrategyResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=constants.USERPATH_STRATEGY, - extra_dir=extra_dir) + abs_paths = StrategyResolver.build_search_paths(config, current_path=current_path, + user_subdir=constants.USERPATH_STRATEGY, + extra_dir=extra_dir) if ":" in strategy_name: logger.info("loading base64 encoded strategy") @@ -154,8 +155,9 @@ class StrategyResolver(IResolver): # register temp path with the bot abs_paths.insert(0, temp.resolve()) - strategy = IResolver._load_object(paths=abs_paths, object_type=IStrategy, - object_name=strategy_name, kwargs={'config': config}) + strategy = StrategyResolver._load_object(paths=abs_paths, + object_name=strategy_name, + kwargs={'config': config}) if strategy: strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index ce7ac1741..dba816621 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -20,7 +20,6 @@ def test_search_strategy(): s, _ = StrategyResolver._search_object( directory=default_location, - object_type=IStrategy, kwargs={'config': default_config}, object_name='DefaultStrategy' ) @@ -28,7 +27,6 @@ def test_search_strategy(): s, _ = StrategyResolver._search_object( directory=default_location, - object_type=IStrategy, kwargs={'config': default_config}, object_name='NotFoundStrategy' ) From 25e6d6a7bfdea9bd74b6f06375c4e11df2f903b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 13:54:46 +0100 Subject: [PATCH 094/128] Combine load_object methods into one --- freqtrade/resolvers/hyperopt_resolver.py | 70 +++++------------------- freqtrade/resolvers/iresolver.py | 36 +++++++++++- freqtrade/resolvers/pairlist_resolver.py | 42 ++++---------- freqtrade/resolvers/strategy_resolver.py | 16 ++++-- tests/optimize/test_hyperopt.py | 4 +- 5 files changed, 68 insertions(+), 100 deletions(-) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index b9c750251..c26fd09f2 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -5,7 +5,7 @@ This module load custom hyperopt """ import logging from pathlib import Path -from typing import Optional, Dict +from typing import Dict from freqtrade import OperationalException from freqtrade.constants import DEFAULT_HYPEROPT_LOSS, USERPATH_HYPEROPTS @@ -21,6 +21,9 @@ class HyperOptResolver(IResolver): This class contains all the logic to load custom hyperopt class """ object_type = IHyperOpt + object_type_str = "Hyperopt" + user_subdir = USERPATH_HYPEROPTS + initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() @staticmethod def load_hyperopt(config: Dict) -> IHyperOpt: @@ -34,8 +37,9 @@ class HyperOptResolver(IResolver): hyperopt_name = config['hyperopt'] - hyperopt = HyperOptResolver._load_hyperopt(hyperopt_name, config, - extra_dir=config.get('hyperopt_path')) + hyperopt = HyperOptResolver.load_object(hyperopt_name, config, + kwargs={'config': config}, + extra_dir=config.get('hyperopt_path')) if not hasattr(hyperopt, 'populate_indicators'): logger.warning("Hyperopt class does not provide populate_indicators() method. " @@ -48,38 +52,15 @@ class HyperOptResolver(IResolver): "Using populate_sell_trend from the strategy.") return hyperopt - @staticmethod - def _load_hyperopt( - hyperopt_name: str, config: Dict, extra_dir: Optional[str] = None) -> IHyperOpt: - """ - Search and loads the specified hyperopt. - :param hyperopt_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given hyperopt - :return: HyperOpt instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - - abs_paths = HyperOptResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) - - hyperopt = HyperOptResolver._load_object(paths=abs_paths, - object_name=hyperopt_name, - kwargs={'config': config}) - if hyperopt: - return hyperopt - raise OperationalException( - f"Impossible to load Hyperopt '{hyperopt_name}'. This class does not exist " - "or contains Python code errors." - ) - class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class """ object_type = IHyperOptLoss + object_type_str = "HyperoptLoss" + user_subdir = USERPATH_HYPEROPTS + initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() @staticmethod def load_hyperoptloss(config: Dict) -> IHyperOptLoss: @@ -92,8 +73,9 @@ class HyperOptLossResolver(IResolver): # default hyperopt loss hyperoptloss_name = config.get('hyperopt_loss') or DEFAULT_HYPEROPT_LOSS - hyperoptloss = HyperOptLossResolver._load_hyperoptloss( - hyperoptloss_name, config, extra_dir=config.get('hyperopt_path')) + hyperoptloss = HyperOptLossResolver.load_object(hyperoptloss_name, + config, kwargs={}, + extra_dir=config.get('hyperopt_path')) # Assign ticker_interval to be used in hyperopt hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) @@ -103,29 +85,3 @@ class HyperOptLossResolver(IResolver): f"Found HyperoptLoss class {hyperoptloss_name} does not " "implement `hyperopt_loss_function`.") return hyperoptloss - - @staticmethod - def _load_hyperoptloss(hyper_loss_name: str, config: Dict, - extra_dir: Optional[str] = None) -> IHyperOptLoss: - """ - Search and loads the specified hyperopt loss class. - :param hyper_loss_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given hyperopt - :return: HyperOptLoss instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - - abs_paths = HyperOptLossResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) - - hyperoptloss = HyperOptLossResolver._load_object(paths=abs_paths, - object_name=hyper_loss_name) - if hyperoptloss: - return hyperoptloss - - raise OperationalException( - f"Impossible to load HyperoptLoss '{hyper_loss_name}'. This class does not exist " - "or contains Python code errors." - ) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 11937c1da..0101e37a3 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -9,6 +9,8 @@ import logging from pathlib import Path from typing import Any, Generator, List, Optional, Tuple, Type, Union +from freqtrade import OperationalException + logger = logging.getLogger(__name__) @@ -18,12 +20,15 @@ class IResolver: """ # Childclasses need to override this object_type: Type[Any] + object_type_str: str + user_subdir: Optional[str] = None + initial_search_path: Path - @staticmethod - def build_search_paths(config, current_path: Path, user_subdir: Optional[str] = None, + @classmethod + def build_search_paths(cls, config, user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: - abs_paths: List[Path] = [current_path] + abs_paths: List[Path] = [cls.initial_search_path] if user_subdir: abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) @@ -105,3 +110,28 @@ class IResolver: logger.warning('Path "%s" does not exist.', _path.resolve()) return None + + @classmethod + def load_object(cls, object_name: str, config: dict, kwargs: dict, + extra_dir: Optional[str] = None) -> Any: + """ + Search and loads the specified object as configured in hte child class. + :param objectname: name of the module to import + :param config: configuration dictionary + :param extra_dir: additional directory to search for the given pairlist + :raises: OperationalException if the class is invalid or does not exist. + :return: Object instance or None + """ + + abs_paths = cls.build_search_paths(config, + user_subdir=cls.user_subdir, + extra_dir=extra_dir) + + pairlist = cls._load_object(paths=abs_paths, object_name=object_name, + kwargs=kwargs) + if pairlist: + return pairlist + raise OperationalException( + f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist " + "or contains Python code errors." + ) diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 00ebc03aa..77db74084 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -6,7 +6,6 @@ This module load custom pairlists import logging from pathlib import Path -from freqtrade import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import IResolver @@ -18,6 +17,9 @@ class PairListResolver(IResolver): This class contains all the logic to load custom PairList class """ object_type = IPairList + object_type_str = "Pairlist" + user_subdir = None + initial_search_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, @@ -32,34 +34,10 @@ class PairListResolver(IResolver): :param pairlist_pos: Position of the pairlist in the list of pairlists :return: initialized Pairlist class """ - - return PairListResolver._load_pairlist(pairlist_name, config, - kwargs={'exchange': exchange, - 'pairlistmanager': pairlistmanager, - 'config': config, - 'pairlistconfig': pairlistconfig, - 'pairlist_pos': pairlist_pos}) - - @staticmethod - def _load_pairlist(pairlist_name: str, config: dict, kwargs: dict) -> IPairList: - """ - Search and loads the specified pairlist. - :param pairlist_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given pairlist - :return: PairList instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() - - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=None, extra_dir=None) - - pairlist = PairListResolver._load_object(paths=abs_paths, - object_name=pairlist_name, - kwargs=kwargs) - if pairlist: - return pairlist - raise OperationalException( - f"Impossible to load Pairlist '{pairlist_name}'. This class does not exist " - "or contains Python code errors." - ) + return PairListResolver.load_object(pairlist_name, config, + kwargs={'exchange': exchange, + 'pairlistmanager': pairlistmanager, + 'config': config, + 'pairlistconfig': pairlistconfig, + 'pairlist_pos': pairlist_pos}, + ) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 654103377..4fd5c586a 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -11,7 +11,9 @@ from inspect import getfullargspec from pathlib import Path from typing import Dict, Optional -from freqtrade import constants, OperationalException +from freqtrade import OperationalException +from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, + USERPATH_STRATEGY) from freqtrade.resolvers import IResolver from freqtrade.strategy.interface import IStrategy @@ -23,6 +25,9 @@ class StrategyResolver(IResolver): This class contains the logic to load custom strategy class """ object_type = IStrategy + object_type_str = "Strategy" + user_subdir = USERPATH_STRATEGY + initial_search_path = Path(__file__).parent.parent.joinpath('strategy').resolve() @staticmethod def load_strategy(config: Optional[Dict] = None) -> IStrategy: @@ -115,11 +120,11 @@ class StrategyResolver(IResolver): @staticmethod def _strategy_sanity_validations(strategy): - if not all(k in strategy.order_types for k in constants.REQUIRED_ORDERTYPES): + if not all(k in strategy.order_types for k in REQUIRED_ORDERTYPES): raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-types mapping is incomplete.") - if not all(k in strategy.order_time_in_force for k in constants.REQUIRED_ORDERTIF): + if not all(k in strategy.order_time_in_force for k in REQUIRED_ORDERTIF): raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-time-in-force mapping is incomplete.") @@ -133,10 +138,9 @@ class StrategyResolver(IResolver): :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ - current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() - abs_paths = StrategyResolver.build_search_paths(config, current_path=current_path, - user_subdir=constants.USERPATH_STRATEGY, + abs_paths = StrategyResolver.build_search_paths(config, + user_subdir=USERPATH_STRATEGY, extra_dir=extra_dir) if ":" in strategy_name: diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 9c6e73c53..fb492be35 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -159,7 +159,7 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: delattr(hyperopt, 'populate_buy_trend') delattr(hyperopt, 'populate_sell_trend') mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt', + 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver.load_object', MagicMock(return_value=hyperopt(default_conf)) ) default_conf.update({'hyperopt': 'DefaultHyperOpt'}) @@ -195,7 +195,7 @@ def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None: hl = DefaultHyperOptLoss mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver._load_hyperoptloss', + 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', MagicMock(return_value=hl) ) x = HyperOptLossResolver.load_hyperoptloss(default_conf) From 5a11ca86bb69b1619cbba78cce05a205f669a67d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 14:01:28 +0100 Subject: [PATCH 095/128] Move instanciation out of search_object --- freqtrade/resolvers/iresolver.py | 13 ++++++------- tests/strategy/test_strategy.py | 5 +---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 0101e37a3..bbdc8ca91 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -66,13 +66,13 @@ class IResolver: return valid_objects_gen @classmethod - def _search_object(cls, directory: Path, object_name: str, - kwargs: dict = {}) -> Union[Tuple[Any, Path], Tuple[None, None]]: + def _search_object(cls, directory: Path, object_name: str + ) -> Union[Tuple[Any, Path], Tuple[None, None]]: """ Search for the objectname in the given directory :param directory: relative or absolute directory path :param object_name: ClassName of the object to load - :return: object instance + :return: object class """ logger.debug("Searching for %s %s in '%s'", cls.object_type.__name__, object_name, directory) @@ -86,7 +86,7 @@ class IResolver: obj = next(cls._get_valid_object(module_path, object_name), None) if obj: - return (obj(**kwargs), module_path) + return (obj, module_path) return (None, None) @classmethod @@ -99,13 +99,12 @@ class IResolver: for _path in paths: try: (module, module_path) = cls._search_object(directory=_path, - object_name=object_name, - kwargs=kwargs) + object_name=object_name) if module: logger.info( f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} " f"from '{module_path}'...") - return module + return module(**kwargs) except FileNotFoundError: logger.warning('Path "%s" does not exist.', _path.resolve()) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index dba816621..7085223c5 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -15,19 +15,16 @@ from tests.conftest import log_has, log_has_re def test_search_strategy(): - default_config = {} default_location = Path(__file__).parent.parent.joinpath('strategy').resolve() s, _ = StrategyResolver._search_object( directory=default_location, - kwargs={'config': default_config}, object_name='DefaultStrategy' ) - assert isinstance(s, IStrategy) + assert issubclass(s, IStrategy) s, _ = StrategyResolver._search_object( directory=default_location, - kwargs={'config': default_config}, object_name='NotFoundStrategy' ) assert s is None From 2ab989e274c5bfb278e8b6397380b1714d806a80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:28:35 +0100 Subject: [PATCH 096/128] Cleanup some code and add option --- freqtrade/configuration/arguments.py | 2 +- freqtrade/resolvers/iresolver.py | 35 +++++++++++++++++++++++----- freqtrade/utils.py | 14 +++++++++-- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 5f7bc74f1..b2197619d 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,7 +30,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] -ARGS_LIST_STRATEGIES = ["strategy_path"] +ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column"] ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index bbdc8ca91..01ecbcb84 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -7,7 +7,7 @@ import importlib.util import inspect import logging from pathlib import Path -from typing import Any, Generator, List, Optional, Tuple, Type, Union +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, Union from freqtrade import OperationalException @@ -41,7 +41,7 @@ class IResolver: @classmethod def _get_valid_object(cls, module_path: Path, - object_name: str) -> Generator[Any, None, None]: + object_name: Optional[str]) -> Generator[Any, None, None]: """ Generator returning objects with matching object_type and object_name in the path given. :param module_path: absolute path to the module @@ -51,7 +51,7 @@ class IResolver: # Generate spec based on absolute path # Pass object_name as first argument to have logging print a reasonable name. - spec = importlib.util.spec_from_file_location(object_name, str(module_path)) + spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints @@ -61,7 +61,7 @@ class IResolver: valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) - if object_name == name and cls.object_type in obj.__bases__ + if (object_name is None or object_name == name) and cls.object_type in obj.__bases__ ) return valid_objects_gen @@ -74,8 +74,7 @@ class IResolver: :param object_name: ClassName of the object to load :return: object class """ - logger.debug("Searching for %s %s in '%s'", - cls.object_type.__name__, object_name, directory) + logger.debug(f"Searching for {cls.object_type.__name__} {object_name} in '{directory}'") for entry in directory.iterdir(): # Only consider python files if not str(entry).endswith('.py'): @@ -134,3 +133,27 @@ class IResolver: f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist " "or contains Python code errors." ) + + @classmethod + def search_all_objects(cls, directory: Path) -> List[Dict[str, Any]]: + """ + Searches a directory for valid objects + :param directory: Path to search + :return: List of dicts containing 'name', 'class' and 'location' entires + """ + logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") + objects = [] + for entry in directory.iterdir(): + # Only consider python files + if not str(entry).endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + module_path = entry.resolve() + logger.info(f"Path {module_path}") + for obj in cls._get_valid_object(module_path, object_name=None): + objects.append( + {'name': obj.__name__, + 'class': obj, + 'location': entry, + }) + return objects diff --git a/freqtrade/utils.py b/freqtrade/utils.py index c5fc47a74..06a62172a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -23,7 +23,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, from freqtrade.exchange import (available_exchanges, ccxt_exchanges, market_is_active, symbol_is_pair) from freqtrade.misc import plural, render_template -from freqtrade.resolvers import ExchangeResolver +from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -229,7 +229,17 @@ def start_list_strategies(args: Dict[str, Any]) -> None: Print Strategies available in a folder """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - print(config) + + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) + strategies = StrategyResolver.search_all_objects(directory) + # Sort alphabetically + strategies = sorted(strategies, key=lambda x: x['name']) + strats_to_print = [{'name': s['name'], 'location': s['location']} for s in strategies] + + if args['print_one_column']: + print('\n'.join([s['name'] for s in strategies])) + else: + print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) def start_list_timeframes(args: Dict[str, Any]) -> None: From 27b86170778c70dd4a83d2c02d06a12452fb1783 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:35:38 +0100 Subject: [PATCH 097/128] Add tests --- tests/strategy/test_strategy.py | 8 +++++++ tests/test_utils.py | 42 +++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 7085223c5..10b9f3466 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -30,6 +30,14 @@ def test_search_strategy(): assert s is None +def test_search_all_strategies(): + directory = Path(__file__).parent + strategies = StrategyResolver.search_all_objects(directory) + assert isinstance(strategies, list) + assert len(strategies) == 3 + assert isinstance(strategies[0], dict) + + def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', 'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates') diff --git a/tests/test_utils.py b/tests/test_utils.py index 40ca9ac02..b8be1ae61 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,11 +7,12 @@ import pytest from freqtrade import OperationalException from freqtrade.state import RunMode from freqtrade.utils import (setup_utils_configuration, start_create_userdir, - start_download_data, start_list_exchanges, - start_list_markets, start_list_timeframes, - start_new_hyperopt, start_new_strategy, - start_test_pairlist, start_trading, - start_hyperopt_list, start_hyperopt_show) + start_download_data, start_hyperopt_list, + start_hyperopt_show, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_hyperopt, + start_new_strategy, start_test_pairlist, + start_trading) from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -630,6 +631,37 @@ def test_download_data_trades(mocker, caplog): assert convert_mock.call_count == 1 +def test_start_list_strategies(mocker, caplog, capsys): + + args = [ + "list-strategies", + "--strategy-path", + str(Path(__file__).parent / "strategy"), + "-1" + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_strategies(pargs) + captured = capsys.readouterr() + assert "TestStrategyLegacy" in captured.out + assert "strategy/legacy_strategy.py" not in captured.out + assert "DefaultStrategy" in captured.out + + # Test regular output + args = [ + "list-strategies", + "--strategy-path", + str(Path(__file__).parent / "strategy"), + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_strategies(pargs) + captured = capsys.readouterr() + assert "TestStrategyLegacy" in captured.out + assert "strategy/legacy_strategy.py" in captured.out + assert "DefaultStrategy" in captured.out + + def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, capsys): mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), From 66f9ece0619554f072fbc20c251b35dd6bbc6f33 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:35:53 +0100 Subject: [PATCH 098/128] Add documentation for strategy-list --- docs/utils.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/utils.md b/docs/utils.md index a9fbfc7d5..6eb37a386 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -108,6 +108,44 @@ With custom user directory freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt ``` +## List Strategies + +Use the `list-strategies` subcommand to see all strategies in one particular folder. + +``` +freqtrade list-strategies --help +usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--strategy-path PATH] [-1] + +optional arguments: + -h, --help show this help message and exit + --strategy-path PATH Specify additional strategy lookup path. + -1, --one-column Print output in one column. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` +Example: search default strategy folder within userdir + +``` bash +freqtrade list-strategies --user-data ~/.freqtrade/ +``` + +Example: search dedicated strategy path + +``` bash +freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ +``` + ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. From 402c761a231edffe017ba93fdf821637e5ddab40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:40:49 +0100 Subject: [PATCH 099/128] Change loglevel of Path output to debug --- docs/utils.md | 5 ++++- freqtrade/resolvers/iresolver.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 6eb37a386..f7501ae9d 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -132,8 +132,11 @@ Common arguments: Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. - ``` + +!!! Warning + Using this command will try to load all python files from a folder. This can be a security risk if untrusted files reside in this folder, since all module-level code is executed. + Example: search default strategy folder within userdir ``` bash diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 01ecbcb84..e3c0d1ad0 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -149,7 +149,7 @@ class IResolver: logger.debug('Ignoring %s', entry) continue module_path = entry.resolve() - logger.info(f"Path {module_path}") + logger.debug(f"Path {module_path}") for obj in cls._get_valid_object(module_path, object_name=None): objects.append( {'name': obj.__name__, From ad75048678796a23e4dcf4a82464628a2860d6b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:53:40 +0100 Subject: [PATCH 100/128] Fix testing with path in windows --- tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index b8be1ae61..185425efc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -644,7 +644,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert "strategy/legacy_strategy.py" not in captured.out + assert str(Path("strategy/legacy_strategy.py")) not in captured.out assert "DefaultStrategy" in captured.out # Test regular output @@ -658,7 +658,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert "strategy/legacy_strategy.py" in captured.out + assert str(Path("strategy/legacy_strategy.py")) in captured.out assert "DefaultStrategy" in captured.out From e5aed098b568900f34e7f3cbe85b8b8d21d76254 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 09:39:29 +0100 Subject: [PATCH 101/128] Enhance backtest results with sell reason profit / loss table --- freqtrade/optimize/backtesting.py | 6 ++++-- tests/optimize/test_backtesting.py | 12 ++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffa112bd5..21fb24b17 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -183,9 +183,11 @@ class Backtesting: Generate small table outlining Backtest results """ tabular_data = [] - headers = ['Sell Reason', 'Count'] + headers = ['Sell Reason', 'Count', 'Profit', 'Loss'] for reason, count in results['sell_reason'].value_counts().iteritems(): - tabular_data.append([reason.value, count]) + profit = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] >= 0)]) + loss = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] < 0)]) + tabular_data.append([reason.value, count, profit, loss]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") def _generate_text_table_strategy(self, all_results: dict) -> str: diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 0ea35e41f..8a27c591f 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -394,8 +394,8 @@ def test_generate_text_table_sell_reason(default_conf, mocker): results = pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], - 'profit_percent': [0.1, 0.2, 0.3], - 'profit_abs': [0.2, 0.4, 0.5], + 'profit_percent': [0.1, 0.2, -0.3], + 'profit_abs': [0.2, 0.4, -0.5], 'trade_duration': [10, 30, 10], 'profit': [2, 0, 0], 'loss': [0, 0, 1], @@ -404,10 +404,10 @@ def test_generate_text_table_sell_reason(default_conf, mocker): ) result_str = ( - '| Sell Reason | Count |\n' - '|:--------------|--------:|\n' - '| roi | 2 |\n' - '| stop_loss | 1 |' + '| Sell Reason | Count | Profit | Loss |\n' + '|:--------------|--------:|---------:|-------:|\n' + '| roi | 2 | 2 | 0 |\n' + '| stop_loss | 1 | 0 | 1 |' ) assert backtesting._generate_text_table_sell_reason( data={'ETH/BTC': {}}, results=results) == result_str From 63f41cf1c692d69da8899fec520916d6e59699b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 09:39:44 +0100 Subject: [PATCH 102/128] Update documentation with new result --- docs/backtesting.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 017289905..ac7c8e11a 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -137,12 +137,12 @@ A backtesting result will look like that: | ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 | 15 | | TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | ========================================================= SELL REASON STATS ========================================================= -| Sell Reason | Count | -|:-------------------|--------:| -| trailing_stop_loss | 205 | -| stop_loss | 166 | -| sell_signal | 56 | -| force_sell | 2 | +| Sell Reason | Count | Profit | Loss | +|:-------------------|--------:|---------:|-------:| +| trailing_stop_loss | 205 | 150 | 55 | +| stop_loss | 166 | 0 | 166 | +| sell_signal | 56 | 36 | 20 | +| force_sell | 2 | 0 | 2 | ====================================================== LEFT OPEN TRADES REPORT ====================================================== | pair | buy count | avg profit % | cum profit % | tot profit BTC | tot profit % | avg duration | profit | loss | |:---------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| @@ -154,6 +154,7 @@ A backtesting result will look like that: The 1st table contains all trades the bot made, including "left open trades". The 2nd table contains a recap of sell reasons. +This table can tell you which area needs some additional work (i.e. all `sell_signal` trades are losses, so we should disable the sell-signal or work on improving that). The 3rd table contains all trades the bot had to `forcesell` at the end of the backtest period to present a full picture. This is necessary to simulate realistic behaviour, since the backtest period has to end at some point, while realistically, you could leave the bot running forever. From 98647b490c6aa311e1b1735ebb5e1cab96020ef1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 19:27:08 +0100 Subject: [PATCH 103/128] Remove wrong "once per hour" listings --- docs/edge.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 769e48927..e7909594e 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -147,8 +147,8 @@ Edge module has following configuration options: |------------|-------------| | `enabled` | If true, then Edge will run periodically.
*Defaults to `false`.*
***Datatype:*** *Boolean* | `process_throttle_secs` | How often should Edge run in seconds.
*Defaults to `3600` (once per hour).*
***Datatype:*** *Integer* -| `calculate_since_number_of_days` | Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy.
**Note** that it downloads historical data so increasing this number would lead to slowing down the bot.
*Defaults to `7` (once per hour).*
***Datatype:*** *Integer* -| `capital_available_percentage` | This is the percentage of the total capital on exchange in stake currency.
As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
*Defaults to `0.5` (once per hour).*
***Datatype:*** *Float* +| `calculate_since_number_of_days` | Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy.
**Note** that it downloads historical data so increasing this number would lead to slowing down the bot.
*Defaults to `7`.*
***Datatype:*** *Integer* +| `capital_available_percentage` | This is the percentage of the total capital on exchange in stake currency.
As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
*Defaults to `0.5`.*
***Datatype:*** *Float* | `allowed_risk` | Ratio of allowed risk per trade.
*Defaults to `0.01` (1%)).*
***Datatype:*** *Float* | `stoploss_range_min` | Minimum stoploss.
*Defaults to `-0.01`.*
***Datatype:*** *Float* | `stoploss_range_max` | Maximum stoploss.
*Defaults to `-0.10`.*
***Datatype:*** *Float* From cadde3ab6d32a207e805f7043598e2c8e0eef3fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 16:15:44 +0100 Subject: [PATCH 104/128] Check if markets.info is a dict before using it --- freqtrade/exchange/exchange.py | 10 +++++++++- tests/exchange/test_exchange.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 384ec1fb7..01e84c06e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -278,7 +278,15 @@ class Exchange: raise OperationalException( f'Pair {pair} is not available on {self.name}. ' f'Please remove {pair} from your whitelist.') - elif self.markets[pair].get('info', {}).get('IsRestricted', False): + + # From ccxt Documentation: + # markets.info: An associative array of non-common market properties, + # including fees, rates, limits and other general market information. + # The internal info array is different for each particular market, + # its contents depend on the exchange. + # It can also be a string or similar ... so we need to verify that first. + elif (isinstance(self.markets[pair].get('info', None), dict) + and self.markets[pair].get('info', {}).get('IsRestricted', False)): # Warn users about restricted pairs in whitelist. # We cannot determine reliably if Users are affected. logger.warning(f"Pair {pair} is restricted for some users on this exchange." diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8f335c16b..4190869ad 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -364,7 +364,8 @@ def test_validate_pairs_restricted(default_conf, mocker, caplog): api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ 'ETH/BTC': {}, 'LTC/BTC': {}, 'NEO/BTC': {}, - 'XRP/BTC': {'info': {'IsRestricted': True}} + 'XRP/BTC': {'info': {'IsRestricted': True}}, + 'NEO/BTC': {'info': 'TestString'}, # info can also be a string ... }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) From e51ac2c97356b5d5d11ed55426a83967ca674ca5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 16:22:41 +0100 Subject: [PATCH 105/128] Remove unavailable pair ... --- tests/exchange/test_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 4190869ad..2f95e4e01 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -363,7 +363,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): def test_validate_pairs_restricted(default_conf, mocker, caplog): api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': {}, 'LTC/BTC': {}, 'NEO/BTC': {}, + 'ETH/BTC': {}, 'LTC/BTC': {}, 'XRP/BTC': {'info': {'IsRestricted': True}}, 'NEO/BTC': {'info': 'TestString'}, # info can also be a string ... }) From 039dfc302ca2c8497aea27282f3ab20f5a0c4343 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 01:34:31 +0300 Subject: [PATCH 106/128] No need to convert pair name --- freqtrade/freqtradebot.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e21d89cd3..a9ce14bef 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -351,7 +351,6 @@ class FreqtradeBot: :param pair: pair for which we want to create a LIMIT_BUY :return: None """ - pair_s = pair.replace('_', '/') stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) time_in_force = self.strategy.order_time_in_force['buy'] @@ -362,10 +361,10 @@ class FreqtradeBot: # Calculate amount buy_limit_requested = self.get_target_bid(pair) - min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) + min_stake_amount = self._get_min_pair_stake_amount(pair, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( - f"Can't open a new trade for {pair_s}: stake amount " + f"Can't open a new trade for {pair}: stake amount " f"is too small ({stake_amount} < {min_stake_amount})" ) return False @@ -388,7 +387,7 @@ class FreqtradeBot: if float(order['filled']) == 0: logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' zero amount is fulfilled.', - order_tif, order_type, pair_s, order_status, self.exchange.name) + order_tif, order_type, pair, order_status, self.exchange.name) return False else: # the order is partially fulfilled @@ -396,7 +395,7 @@ class FreqtradeBot: # if the order is fulfilled fully or partially logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' %s amount fulfilled out of %s (%s remaining which is canceled).', - order_tif, order_type, pair_s, order_status, self.exchange.name, + order_tif, order_type, pair, order_status, self.exchange.name, order['filled'], order['amount'], order['remaining'] ) stake_amount = order['cost'] @@ -413,7 +412,7 @@ class FreqtradeBot: self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), - 'pair': pair_s, + 'pair': pair, 'limit': buy_limit_filled_price, 'order_type': order_type, 'stake_amount': stake_amount, From b6d1c5b17aca2e21d6cde9fd41513a5daf3aa0a2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 01:44:51 +0300 Subject: [PATCH 107/128] _get_trade_stake_amount() is not private --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/rpc/rpc.py | 2 +- tests/test_freqtradebot.py | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a9ce14bef..9bd7e94fc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -204,7 +204,7 @@ class FreqtradeBot: return used_rate - def _get_trade_stake_amount(self, pair) -> Optional[float]: + def get_trade_stake_amount(self, pair) -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance for the stake currency @@ -309,7 +309,7 @@ class FreqtradeBot: self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval)) if buy and not sell and len(Trade.get_open_trades()) < self.config['max_open_trades']: - stake_amount = self._get_trade_stake_amount(_pair) + stake_amount = self.get_trade_stake_amount(_pair) if not stake_amount: continue diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index d6d442df5..35c312743 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -462,7 +462,7 @@ class RPC: raise RPCException(f'position for {pair} already open - id: {trade.id}') # gen stake amount - stakeamount = self._freqtrade._get_trade_stake_amount(pair) + stakeamount = self._freqtrade.get_trade_stake_amount(pair) # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 13f1277b9..1d6cce7fa 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -136,7 +136,7 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: freqtrade = FreqtradeBot(default_conf) - result = freqtrade._get_trade_stake_amount('ETH/BTC') + result = freqtrade.get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] @@ -147,7 +147,7 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade._get_trade_stake_amount('ETH/BTC') + freqtrade.get_trade_stake_amount('ETH/BTC') def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, @@ -170,25 +170,25 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, patch_get_signal(freqtrade) # no open trades, order amount should be 'balance / max_open_trades' - result = freqtrade._get_trade_stake_amount('ETH/BTC') + result = freqtrade.get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] / conf['max_open_trades'] # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' freqtrade.execute_buy('ETH/BTC', result) - result = freqtrade._get_trade_stake_amount('LTC/BTC') + result = freqtrade.get_trade_stake_amount('LTC/BTC') assert result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1) # create 2 trades, order amount should be None freqtrade.execute_buy('LTC/BTC', result) - result = freqtrade._get_trade_stake_amount('XRP/BTC') + result = freqtrade.get_trade_stake_amount('XRP/BTC') assert result is None # set max_open_trades = None, so do not trade conf['max_open_trades'] = 0 freqtrade = FreqtradeBot(conf) - result = freqtrade._get_trade_stake_amount('NEO/BTC') + result = freqtrade.get_trade_stake_amount('NEO/BTC') assert result is None @@ -214,8 +214,8 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: edge_conf['dry_run_wallet'] = 999.9 freqtrade = FreqtradeBot(edge_conf) - assert freqtrade._get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20 - assert freqtrade._get_trade_stake_amount('LTC/BTC') == (999.9 * 0.5 * 0.01) / 0.21 + assert freqtrade.get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20 + assert freqtrade.get_trade_stake_amount('LTC/BTC') == (999.9 * 0.5 * 0.01) / 0.21 def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf) -> None: @@ -570,7 +570,7 @@ def test_create_trades_limit_reached(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) assert not freqtrade.create_trades() - assert freqtrade._get_trade_stake_amount('ETH/BTC') is None + assert freqtrade.get_trade_stake_amount('ETH/BTC') is None def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, From 86f269304099489279934f5622205bdad159d01e Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 01:54:12 +0300 Subject: [PATCH 108/128] cosmetics --- freqtrade/freqtradebot.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9bd7e94fc..7827f29af 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -299,17 +299,17 @@ class FreqtradeBot: buycount = 0 # running get_signal on historical data fetched - for _pair in whitelist: - if self.strategy.is_pair_locked(_pair): - logger.info(f"Pair {_pair} is currently locked.") + for pair in whitelist: + if self.strategy.is_pair_locked(pair): + logger.info(f"Pair {pair} is currently locked.") continue (buy, sell) = self.strategy.get_signal( - _pair, self.strategy.ticker_interval, - self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval)) + pair, self.strategy.ticker_interval, + self.dataprovider.ohlcv(pair, self.strategy.ticker_interval)) if buy and not sell and len(Trade.get_open_trades()) < self.config['max_open_trades']: - stake_amount = self.get_trade_stake_amount(_pair) + stake_amount = self.get_trade_stake_amount(pair) if not stake_amount: continue @@ -320,11 +320,11 @@ class FreqtradeBot: get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): - if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market): - buycount += self.execute_buy(_pair, stake_amount) + if self._check_depth_of_market_buy(pair, bidstrat_check_depth_of_market): + buycount += self.execute_buy(pair, stake_amount) continue - buycount += self.execute_buy(_pair, stake_amount) + buycount += self.execute_buy(pair, stake_amount) return buycount > 0 From 243bcb23680f5989dce55c268240b074de9448fd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 02:25:43 +0300 Subject: [PATCH 109/128] Make _check_available_stake_amount() a separate method --- freqtrade/freqtradebot.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7827f29af..24d250ffe 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -206,9 +206,8 @@ class FreqtradeBot: def get_trade_stake_amount(self, pair) -> Optional[float]: """ - Check if stake amount can be fulfilled with the available balance - for the stake currency - :return: float: Stake Amount + Calculate stake amount for the trade + :return: float: Stake amount """ if self.edge: return self.edge.stake_amount( @@ -220,16 +219,24 @@ class FreqtradeBot: else: stake_amount = self.config['stake_amount'] - available_amount = self.wallets.get_free(self.config['stake_currency']) - if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.get_open_trades()) if open_trades >= self.config['max_open_trades']: logger.warning("Can't open a new trade: max number of trades is reached") return None + available_amount = self.wallets.get_free(self.config['stake_currency']) return available_amount / (self.config['max_open_trades'] - open_trades) - # Check if stake_amount is fulfilled + return self._check_available_stake_amount(stake_amount) + + def _check_available_stake_amount(self, stake_amount) -> float: + """ + Check if stake amount can be fulfilled with the available balance + for the stake currency + :return: float: Stake amount + """ + available_amount = self.wallets.get_free(self.config['stake_currency']) + if available_amount < stake_amount: raise DependencyException( f"Available balance ({available_amount} {self.config['stake_currency']}) is " From abaeab89aadee4f8a6ee3a29d381fa58261fe0b1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 02:36:32 +0300 Subject: [PATCH 110/128] Make _calculate_unlimited_stake_amount() a separate method --- freqtrade/freqtradebot.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 24d250ffe..7b111e2d4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -220,15 +220,22 @@ class FreqtradeBot: stake_amount = self.config['stake_amount'] if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: - open_trades = len(Trade.get_open_trades()) - if open_trades >= self.config['max_open_trades']: - logger.warning("Can't open a new trade: max number of trades is reached") - return None - available_amount = self.wallets.get_free(self.config['stake_currency']) - return available_amount / (self.config['max_open_trades'] - open_trades) + return self._calculate_unlimited_stake_amount() return self._check_available_stake_amount(stake_amount) + def _calculate_unlimited_stake_amount(self) -> Optional[float]: + """ + Calculate stake amount for "unlimited" stake amount + :return: None if max number of trades reached + """ + open_trades = len(Trade.get_open_trades()) + if open_trades >= self.config['max_open_trades']: + logger.warning("Can't open a new trade: max number of trades is reached") + return None + available_amount = self.wallets.get_free(self.config['stake_currency']) + return available_amount / (self.config['max_open_trades'] - open_trades) + def _check_available_stake_amount(self, stake_amount) -> float: """ Check if stake amount can be fulfilled with the available balance From ef92fd775c0863530caae52e9b0b170c3fcc7b77 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 02:53:41 +0300 Subject: [PATCH 111/128] Align behavior: check for available in all cases: edge, unlimited and fixed --- freqtrade/freqtradebot.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7b111e2d4..2f4803cc5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -210,7 +210,7 @@ class FreqtradeBot: :return: float: Stake amount """ if self.edge: - return self.edge.stake_amount( + stake_amount = self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), self.wallets.get_total(self.config['stake_currency']), @@ -218,9 +218,8 @@ class FreqtradeBot: ) else: stake_amount = self.config['stake_amount'] - - if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: - return self._calculate_unlimited_stake_amount() + if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: + stake_amount = self._calculate_unlimited_stake_amount() return self._check_available_stake_amount(stake_amount) @@ -236,7 +235,7 @@ class FreqtradeBot: available_amount = self.wallets.get_free(self.config['stake_currency']) return available_amount / (self.config['max_open_trades'] - open_trades) - def _check_available_stake_amount(self, stake_amount) -> float: + def _check_available_stake_amount(self, stake_amount: Optional[float]) -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance for the stake currency @@ -244,7 +243,7 @@ class FreqtradeBot: """ available_amount = self.wallets.get_free(self.config['stake_currency']) - if available_amount < stake_amount: + if stake_amount is not None and available_amount < stake_amount: raise DependencyException( f"Available balance ({available_amount} {self.config['stake_currency']}) is " f"lower than stake amount ({stake_amount} {self.config['stake_currency']})" From ed9cb4219df4305e5c3aa1133b45ca0637b9b6d4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 02:58:23 +0300 Subject: [PATCH 112/128] Make mypy happy --- freqtrade/freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2f4803cc5..4ec01cb60 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -209,6 +209,7 @@ class FreqtradeBot: Calculate stake amount for the trade :return: float: Stake amount """ + stake_amount: Optional[float] if self.edge: stake_amount = self.edge.stake_amount( pair, From 8eeabd2372b26987975a4dd772157e14de7a0044 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 03:22:50 +0300 Subject: [PATCH 113/128] Move warning to create_trades() --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4ec01cb60..44a83db1a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -231,7 +231,6 @@ class FreqtradeBot: """ open_trades = len(Trade.get_open_trades()) if open_trades >= self.config['max_open_trades']: - logger.warning("Can't open a new trade: max number of trades is reached") return None available_amount = self.wallets.get_free(self.config['stake_currency']) return available_amount / (self.config['max_open_trades'] - open_trades) @@ -324,7 +323,8 @@ class FreqtradeBot: if buy and not sell and len(Trade.get_open_trades()) < self.config['max_open_trades']: stake_amount = self.get_trade_stake_amount(pair) - if not stake_amount: + if stake_amount is None: + logger.warning("Can't open a new trade: max number of trades is reached") continue logger.info(f"Buy signal found: about create a new trade with stake_amount: " From 3dbd83e35a7529215ef609e3090aa9d78475ecb4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 03:46:42 +0300 Subject: [PATCH 114/128] Introduce get_free_open_trades() method --- freqtrade/freqtradebot.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 44a83db1a..14d67b942 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -136,7 +136,7 @@ class FreqtradeBot: self.process_maybe_execute_sells(trades) # Then looking for buy opportunities - if len(trades) < self.config['max_open_trades']: + if self.get_free_open_trades(): self.process_maybe_execute_buys() # Check and handle any timed out open orders @@ -173,6 +173,14 @@ class FreqtradeBot: """ return [(pair, self.config['ticker_interval']) for pair in pairs] + def get_free_open_trades(self): + """ + Return the number of free open trades slots or 0 if + max number of open trades reached + """ + open_trades = len(Trade.get_open_trades()) + return max(0, self.config['max_open_trades'] - open_trades) + def get_target_bid(self, pair: str, tick: Dict = None) -> float: """ Calculates bid target between current ask price and last price @@ -229,11 +237,11 @@ class FreqtradeBot: Calculate stake amount for "unlimited" stake amount :return: None if max number of trades reached """ - open_trades = len(Trade.get_open_trades()) - if open_trades >= self.config['max_open_trades']: + free_open_trades = self.get_free_open_trades() + if not free_open_trades: return None available_amount = self.wallets.get_free(self.config['stake_currency']) - return available_amount / (self.config['max_open_trades'] - open_trades) + return available_amount / free_open_trades def _check_available_stake_amount(self, stake_amount: Optional[float]) -> Optional[float]: """ @@ -321,12 +329,12 @@ class FreqtradeBot: pair, self.strategy.ticker_interval, self.dataprovider.ohlcv(pair, self.strategy.ticker_interval)) - if buy and not sell and len(Trade.get_open_trades()) < self.config['max_open_trades']: - stake_amount = self.get_trade_stake_amount(pair) - if stake_amount is None: + if buy and not sell: + if not self.get_free_open_trades(): logger.warning("Can't open a new trade: max number of trades is reached") continue + stake_amount = self.get_trade_stake_amount(pair) logger.info(f"Buy signal found: about create a new trade with stake_amount: " f"{stake_amount} ...") From d6ca562b0378a289b160372d1a4a7df0d2da2d67 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 04:05:03 +0300 Subject: [PATCH 115/128] Make mypy happy and handle hypothetical case when stake_amount == 0 --- freqtrade/freqtradebot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 14d67b942..9c18d2d95 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -335,6 +335,10 @@ class FreqtradeBot: continue stake_amount = self.get_trade_stake_amount(pair) + if not stake_amount: + logger.warning("Stake amount is 0, ignoring trade") + continue + logger.info(f"Buy signal found: about create a new trade with stake_amount: " f"{stake_amount} ...") From fc98cf00372b8b5f0d8e9cdd6ea8dc3b182f3f4a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 06:25:45 +0100 Subject: [PATCH 116/128] Address PR feedback - change output to show Filename only --- docs/utils.md | 8 ++++---- freqtrade/utils.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index f7501ae9d..18deeac54 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -110,7 +110,7 @@ freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt ## List Strategies -Use the `list-strategies` subcommand to see all strategies in one particular folder. +Use the `list-strategies` subcommand to see all strategies in one particular directory. ``` freqtrade list-strategies --help @@ -135,12 +135,12 @@ Common arguments: ``` !!! Warning - Using this command will try to load all python files from a folder. This can be a security risk if untrusted files reside in this folder, since all module-level code is executed. + Using this command will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. -Example: search default strategy folder within userdir +Example: search default strategy directory within userdir ``` bash -freqtrade list-strategies --user-data ~/.freqtrade/ +freqtrade list-strategies --userdir ~/.freqtrade/ ``` Example: search dedicated strategy path diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 06a62172a..f6e251154 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -226,7 +226,7 @@ def start_download_data(args: Dict[str, Any]) -> None: def start_list_strategies(args: Dict[str, Any]) -> None: """ - Print Strategies available in a folder + Print Strategies available in a directory """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -234,7 +234,7 @@ def start_list_strategies(args: Dict[str, Any]) -> None: strategies = StrategyResolver.search_all_objects(directory) # Sort alphabetically strategies = sorted(strategies, key=lambda x: x['name']) - strats_to_print = [{'name': s['name'], 'location': s['location']} for s in strategies] + strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] if args['print_one_column']: print('\n'.join([s['name'] for s in strategies])) From b2fb28453f460f89bfaef57a2ac5edae90749585 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 06:39:25 +0100 Subject: [PATCH 117/128] Fix tests after changing output --- tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 185425efc..4cf7b5f23 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -644,7 +644,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert str(Path("strategy/legacy_strategy.py")) not in captured.out + assert "legacy_strategy.py" not in captured.out assert "DefaultStrategy" in captured.out # Test regular output @@ -658,7 +658,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert str(Path("strategy/legacy_strategy.py")) in captured.out + assert "legacy_strategy.py" in captured.out assert "DefaultStrategy" in captured.out From 5c39ebd0a0d9dea455762086d78a3ea5cbf3c995 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 28 Dec 2019 13:59:40 +0300 Subject: [PATCH 118/128] Adjust logging --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9c18d2d95..1cc7f32f4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -331,12 +331,12 @@ class FreqtradeBot: if buy and not sell: if not self.get_free_open_trades(): - logger.warning("Can't open a new trade: max number of trades is reached") + logger.debug("Can't open a new trade: max number of trades is reached") continue stake_amount = self.get_trade_stake_amount(pair) if not stake_amount: - logger.warning("Stake amount is 0, ignoring trade") + logger.debug("Stake amount is 0, ignoring possible trade for {pair}.") continue logger.info(f"Buy signal found: about create a new trade with stake_amount: " From d1c45cf3f869968523f5b84e7949efd8130e166d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Dec 2019 13:07:51 +0100 Subject: [PATCH 119/128] Update analysis documentation to include kernel installation --- docs/data-analysis.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 115ce1916..8786a1199 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -8,6 +8,27 @@ You can analyze the results of backtests and trading history easily using Jupyte * Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)* * Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update. +### Using virtual environment with System jupyter + +Sometimes it can be desired to use a system-wide installation of Jupyter notebook, and use a jupyter kernel from the virtual environment. +This prevents you from installing the full jupyter suite multiple times per system, and provides an easy way to switch between tasks (freqtrade / other analytics tasks). + +For this to work, first activate your virtual environment and run the following commands: + +``` bash +# Activate virtual environment +source .env/bin/activate + +pip install ipykernel +ipython kernel install --user --name=freqtrade +# Restart jupyter (lab / notebook) +# select kernel "freqtrade" in the notebook +``` + +!!! Note + This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community). + + ## Fine print Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually. From 304d15e23694d1dc3ac1f85760dbc3af9996710a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Dec 2019 19:35:42 +0100 Subject: [PATCH 120/128] Small corrections --- docs/data-analysis.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 8786a1199..fc4693b17 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -8,7 +8,7 @@ You can analyze the results of backtests and trading history easily using Jupyte * Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)* * Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update. -### Using virtual environment with System jupyter +### Using virtual environment with system-wide Jupyter installation Sometimes it can be desired to use a system-wide installation of Jupyter notebook, and use a jupyter kernel from the virtual environment. This prevents you from installing the full jupyter suite multiple times per system, and provides an easy way to switch between tasks (freqtrade / other analytics tasks). @@ -26,7 +26,7 @@ ipython kernel install --user --name=freqtrade ``` !!! Note - This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community). + This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community). ## Fine print From df7ceb4ccbbfd35fe7eaec2d37dea273b89a4873 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Dec 2019 19:51:47 +0100 Subject: [PATCH 121/128] Fix misinformation in /status table --- freqtrade/rpc/rpc.py | 2 +- tests/rpc/test_rpc.py | 2 +- tests/rpc/test_rpc_telegram.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 35c312743..0a79d350b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -142,7 +142,7 @@ class RPC: def _rpc_status_table(self, stake_currency, fiat_display_currency: str) -> Tuple[List, List]: trades = Trade.get_open_trades() if not trades: - raise RPCException('no active order') + raise RPCException('no active trade') else: trades_list = [] for trade in trades: diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 3b897572c..c5bb0ca2c 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -113,7 +113,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING - with pytest.raises(RPCException, match=r'.*no active order*'): + with pytest.raises(RPCException, match=r'.*no active trade*'): rpc._rpc_status_table(default_conf['stake_currency'], 'USD') freqtradebot.create_trades() diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index f8b9ca8ab..ddbc35bd5 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -275,13 +275,13 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: # Status table is also enabled when stopped telegram._status_table(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert 'no active order' in msg_mock.call_args_list[0][0][0] + assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() freqtradebot.state = State.RUNNING telegram._status_table(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert 'no active order' in msg_mock.call_args_list[0][0][0] + assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() # Create some test data From 20a132651f460b3ab00f4e4032173b8c58eb7bba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Dec 2019 07:34:49 +0000 Subject: [PATCH 122/128] Bump ccxt from 1.21.12 to 1.21.23 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.21.12 to 1.21.23. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.21.12...1.21.23) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 9d0fd4756..add1ea0fd 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.21.12 +ccxt==1.21.23 SQLAlchemy==1.3.12 python-telegram-bot==12.2.0 arrow==0.15.4 From de23f3928de5639a01efe424e5040354196c59e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Dec 2019 09:56:42 +0100 Subject: [PATCH 123/128] Add trailing_only_offset to template and sample --- freqtrade/templates/base_strategy.py.j2 | 1 + freqtrade/templates/sample_strategy.py | 1 + 2 files changed, 2 insertions(+) diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 73a4c7a5a..32573ec9e 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -47,6 +47,7 @@ class {{ strategy }}(IStrategy): # Trailing stoploss trailing_stop = False + # trailing_only_offset_is_reached = False # trailing_stop_positive = 0.01 # trailing_stop_positive_offset = 0.0 # Disabled / not configured diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 02bf24e7e..228e56b82 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -48,6 +48,7 @@ class SampleStrategy(IStrategy): # Trailing stoploss trailing_stop = False + # trailing_only_offset_is_reached = False # trailing_stop_positive = 0.01 # trailing_stop_positive_offset = 0.0 # Disabled / not configured From fb3a53b8af27c49eb4c8a26c271437a581dd187e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Dec 2019 14:28:34 +0100 Subject: [PATCH 124/128] Use ExchangeResolver for edge_cli too --- freqtrade/optimize/edge_cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index ea5cc663d..4944f1dbb 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -12,8 +12,7 @@ from freqtrade import constants from freqtrade.configuration import (TimeRange, remove_credentials, validate_config_consistency) from freqtrade.edge import Edge -from freqtrade.exchange import Exchange -from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers import StrategyResolver, ExchangeResolver logger = logging.getLogger(__name__) @@ -33,7 +32,7 @@ class EdgeCli: # Reset keys for edge remove_credentials(self.config) self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT - self.exchange = Exchange(self.config) + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.strategy = StrategyResolver.load_strategy(self.config) validate_config_consistency(self.config) From 20abf67779a1c9f778e6339df0abd0af63433154 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Dec 2019 14:29:36 +0100 Subject: [PATCH 125/128] Add Debug "code" for randomly failing test --- tests/test_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4cf7b5f23..8545eb817 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -448,6 +448,9 @@ def test_create_datadir(caplog, mocker): # Ensure that caplog is empty before starting ... # Should prevent random failures. caplog.clear() + # Added assert here to analyze random test-failures ... + assert len(caplog.record_tuples) == 0 + cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock()) csf = mocker.patch("freqtrade.utils.copy_sample_files", MagicMock()) args = [ From 024aa3ab6b6e5e0ba11bf7cfe0035cb83423bbf3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Dec 2019 14:57:26 +0100 Subject: [PATCH 126/128] Move exceptions to seperate module --- freqtrade/__init__.py | 31 ------------------------------- freqtrade/exceptions.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 31 deletions(-) create mode 100644 freqtrade/exceptions.py diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 83fee0b0d..e1f65d4fe 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -11,34 +11,3 @@ if __version__ == 'develop': except Exception: # git not available, ignore pass - - -class DependencyException(Exception): - """ - Indicates that an assumed dependency is not met. - This could happen when there is currently not enough money on the account. - """ - - -class OperationalException(Exception): - """ - Requires manual intervention and will usually stop the bot. - This happens when an exchange returns an unexpected error during runtime - or given configuration is invalid. - """ - - -class InvalidOrderException(Exception): - """ - This is returned when the order is not valid. Example: - If stoploss on exchange order is hit, then trying to cancel the order - should return this exception. - """ - - -class TemporaryError(Exception): - """ - Temporary network or exchange related error. - This could happen when an exchange is congested, unavailable, or the user - has networking problems. Usually resolves itself after a time. - """ diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py new file mode 100644 index 000000000..80d45dd86 --- /dev/null +++ b/freqtrade/exceptions.py @@ -0,0 +1,31 @@ + + +class DependencyException(Exception): + """ + Indicates that an assumed dependency is not met. + This could happen when there is currently not enough money on the account. + """ + + +class OperationalException(Exception): + """ + Requires manual intervention and will usually stop the bot. + This happens when an exchange returns an unexpected error during runtime + or given configuration is invalid. + """ + + +class InvalidOrderException(Exception): + """ + This is returned when the order is not valid. Example: + If stoploss on exchange order is hit, then trying to cancel the order + should return this exception. + """ + + +class TemporaryError(Exception): + """ + Temporary network or exchange related error. + This could happen when an exchange is congested, unavailable, or the user + has networking problems. Usually resolves itself after a time. + """ From 1ffda29fd2aeb95cffedfaded274c4d07e404436 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Dec 2019 15:02:17 +0100 Subject: [PATCH 127/128] Adjust improts to new exception location --- freqtrade/configuration/check_exchange.py | 4 ++-- freqtrade/configuration/config_validation.py | 3 ++- freqtrade/configuration/configuration.py | 5 +++-- freqtrade/configuration/deprecated_settings.py | 2 +- freqtrade/configuration/directory_operations.py | 2 +- freqtrade/configuration/load_config.py | 2 +- freqtrade/data/history.py | 6 ++++-- freqtrade/edge/__init__.py | 4 ++-- freqtrade/exchange/binance.py | 4 ++-- freqtrade/exchange/common.py | 2 +- freqtrade/exchange/exchange.py | 4 ++-- freqtrade/exchange/kraken.py | 2 +- freqtrade/freqtradebot.py | 6 +++--- freqtrade/loggers.py | 2 +- freqtrade/main.py | 2 +- freqtrade/optimize/__init__.py | 4 ++-- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 6 +++--- freqtrade/optimize/hyperopt_interface.py | 6 ++---- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/pairlist/pairlistmanager.py | 5 +++-- freqtrade/persistence.py | 2 +- freqtrade/plot/plot_utils.py | 2 +- freqtrade/resolvers/hyperopt_resolver.py | 2 +- freqtrade/resolvers/iresolver.py | 2 +- freqtrade/resolvers/strategy_resolver.py | 2 +- freqtrade/rpc/rpc.py | 2 +- freqtrade/utils.py | 2 +- freqtrade/worker.py | 4 ++-- tests/edge/test_edge.py | 2 +- tests/exchange/test_binance.py | 4 ++-- tests/exchange/test_exchange.py | 4 ++-- tests/optimize/test_backtesting.py | 4 ++-- tests/optimize/test_hyperopt.py | 2 +- tests/pairlist/test_pairlist.py | 2 +- tests/rpc/test_rpc.py | 4 ++-- tests/strategy/test_strategy.py | 2 +- tests/test_configuration.py | 2 +- tests/test_directory_operations.py | 2 +- tests/test_freqtradebot.py | 10 +++++----- tests/test_main.py | 2 +- tests/test_persistence.py | 3 ++- tests/test_plotting.py | 6 +++--- tests/test_utils.py | 2 +- 44 files changed, 74 insertions(+), 70 deletions(-) diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index c739de692..0076b1c5d 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -1,9 +1,9 @@ import logging from typing import Any, Dict -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, - is_exchange_known_ccxt, is_exchange_bad, + is_exchange_bad, is_exchange_known_ccxt, is_exchange_officially_supported) from freqtrade.state import RunMode diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 068364884..43eead46a 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -4,7 +4,8 @@ from typing import Any, Dict from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match -from freqtrade import constants, OperationalException +from freqtrade import constants +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f73b52c10..a8b7638c8 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -7,15 +7,16 @@ from copy import deepcopy from pathlib import Path from typing import Any, Callable, Dict, List, Optional -from freqtrade import OperationalException, constants +from freqtrade import constants from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings from freqtrade.configuration.directory_operations import (create_datadir, create_userdata_dir) from freqtrade.configuration.load_config import load_config_file +from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts, json_load -from freqtrade.state import RunMode, TRADING_MODES, NON_UTIL_MODES +from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index b1e3535a3..260aae419 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -5,7 +5,7 @@ Functions to handle deprecated settings import logging from typing import Any, Dict -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 556f27742..43a209483 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path from typing import Any, Dict, Optional -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.constants import USER_DATA_FILES logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 7a3ca1798..19179c6c3 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -6,7 +6,7 @@ import logging import sys from typing import Any, Dict -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4c5c0521f..30d168f78 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -16,10 +16,12 @@ from typing import Any, Dict, List, Optional, Tuple import arrow from pandas import DataFrame -from freqtrade import OperationalException, misc +from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import (Exchange, timeframe_to_minutes, + timeframe_to_seconds) logger = logging.getLogger(__name__) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 9ad2485ef..19d65d9d7 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -8,12 +8,12 @@ import numpy as np import utils_find_1st as utf1st from pandas import DataFrame -from freqtrade import constants, OperationalException +from freqtrade import constants from freqtrade.configuration import TimeRange from freqtrade.data import history +from freqtrade.exceptions import OperationalException from freqtrade.strategy.interface import SellType - logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index b5507981f..96f72fcf5 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -4,8 +4,8 @@ from typing import Dict import ccxt -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index ed30b95c7..b38ed35a3 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -1,6 +1,6 @@ import logging -from freqtrade import DependencyException, TemporaryError +from freqtrade.exceptions import DependencyException, TemporaryError logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 01e84c06e..3ef32db62 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -17,9 +17,9 @@ import ccxt.async_support as ccxt_async from ccxt.base.decimal_to_precision import ROUND_DOWN, ROUND_UP from pandas import DataFrame -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.misc import deep_merge_dicts diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index f548489bc..9bcd9cc1f 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -4,7 +4,7 @@ from typing import Dict import ccxt -from freqtrade import OperationalException, TemporaryError +from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.exchange import retrier diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 90eb7beba..f33a8cd03 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -12,17 +12,17 @@ from typing import Any, Dict, List, Optional, Tuple import arrow from requests.exceptions import RequestException -from freqtrade import (DependencyException, InvalidOrderException, __version__, - constants, persistence) +from freqtrade import __version__, constants, persistence from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge +from freqtrade.exceptions import DependencyException, InvalidOrderException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index 27f16ecc3..c69388430 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -5,7 +5,7 @@ from logging import Formatter from logging.handlers import RotatingFileHandler, SysLogHandler from typing import Any, Dict, List -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) diff --git a/freqtrade/main.py b/freqtrade/main.py index 7afaeb1a2..62d2fc05d 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -13,8 +13,8 @@ if sys.version_info < (3, 6): import logging from typing import Any, List -from freqtrade import OperationalException from freqtrade.configuration import Arguments +from freqtrade.exceptions import OperationalException logger = logging.getLogger('freqtrade') diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 1f2f588ef..34760372f 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,11 +1,11 @@ import logging from typing import Any, Dict -from freqtrade import DependencyException, constants, OperationalException +from freqtrade import constants +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode from freqtrade.utils import setup_utils_configuration - logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a8fe90a06..9bd0327e0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -12,11 +12,11 @@ from typing import Any, Dict, List, NamedTuple, Optional from pandas import DataFrame from tabulate import tabulate -from freqtrade import OperationalException from freqtrade.configuration import (TimeRange, remove_credentials, validate_config_consistency) from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 48f883ac5..a8f1a71ef 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,13 +22,13 @@ from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) from pandas import DataFrame -from freqtrade import OperationalException from freqtrade.data.history import get_timerange, trim_dataframe +from freqtrade.exceptions import OperationalException from freqtrade.misc import plural, round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules -from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F4 -from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F4 +from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 +from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, HyperOptResolver) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 856f3eee7..d7d917c19 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -4,17 +4,15 @@ This module defines the interface to apply for hyperopt """ import logging import math - from abc import ABC -from typing import Dict, Any, Callable, List +from typing import Any, Callable, Dict, List from skopt.space import Categorical, Dimension, Integer, Real -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict - logger = logging.getLogger(__name__) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 2df9ba691..4ac9935ba 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -8,7 +8,7 @@ import logging from datetime import datetime from typing import Dict, List -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 1530710d2..55828c6ef 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -4,11 +4,12 @@ Static List provider Provides lists as configured in config.json """ -from cachetools import TTLCache, cached import logging from typing import Dict, List -from freqtrade import OperationalException +from cachetools import TTLCache, cached + +from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 993b68bc7..75116f1e3 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -16,7 +16,7 @@ from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/plot/plot_utils.py index 8de0eb9e7..9eff08396 100644 --- a/freqtrade/plot/plot_utils.py +++ b/freqtrade/plot/plot_utils.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from freqtrade.utils import setup_utils_configuration diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index c26fd09f2..ddf461252 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -7,8 +7,8 @@ import logging from pathlib import Path from typing import Dict -from freqtrade import OperationalException from freqtrade.constants import DEFAULT_HYPEROPT_LOSS, USERPATH_HYPEROPTS +from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt_interface import IHyperOpt from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss from freqtrade.resolvers import IResolver diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index e3c0d1ad0..5a844097c 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -9,7 +9,7 @@ import logging from pathlib import Path from typing import Any, Dict, Generator, List, Optional, Tuple, Type, Union -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 4fd5c586a..9e64f38df 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -11,9 +11,9 @@ from inspect import getfullargspec from pathlib import Path from typing import Dict, Optional -from freqtrade import OperationalException from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGY) +from freqtrade.exceptions import OperationalException from freqtrade.resolvers import IResolver from freqtrade.strategy.interface import IStrategy diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0a79d350b..c187dae4f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple import arrow from numpy import NAN, mean -from freqtrade import DependencyException, TemporaryError +from freqtrade.exceptions import DependencyException, TemporaryError from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.rpc.fiat_convert import CryptoToFiatConverter diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 5a5662e4b..45520ecf7 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -11,7 +11,6 @@ import rapidjson from colorama import init as colorama_init from tabulate import tabulate -from freqtrade import OperationalException from freqtrade.configuration import (Configuration, TimeRange, remove_credentials) from freqtrade.configuration.directory_operations import (copy_sample_files, @@ -20,6 +19,7 @@ from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) +from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, ccxt_exchanges, market_is_active, symbol_is_pair) from freqtrade.misc import plural, render_template diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 8e4be9d43..22651d269 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -8,9 +8,9 @@ from typing import Any, Callable, Dict, Optional import sdnotify -from freqtrade import (OperationalException, TemporaryError, __version__, - constants) +from freqtrade import __version__, constants from freqtrade.configuration import Configuration +from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot from freqtrade.rpc import RPCMessageType from freqtrade.state import State diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index 3a866c0a8..ef1280fa4 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -10,7 +10,7 @@ import numpy as np import pytest from pandas import DataFrame, to_datetime -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.strategy.interface import SellType diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 7720a7d2e..0a12c1cb1 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock import ccxt import pytest -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from tests.conftest import get_patched_exchange diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 2f95e4e01..cb40bdbd9 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -11,8 +11,8 @@ import ccxt import pytest from pandas import DataFrame -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Binance, Exchange, Kraken from freqtrade.exchange.common import API_RETRY_COUNT from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair, diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 8a27c591f..4e2fd01cf 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -10,13 +10,14 @@ import pandas as pd import pytest from arrow import Arrow -from freqtrade import DependencyException, OperationalException, constants +from freqtrade import constants from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize import setup_configuration, start_backtesting from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode @@ -25,7 +26,6 @@ from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) - ORDER_TYPES = [ { 'buy': 'limit', diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index fb492be35..473d760ac 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -9,7 +9,7 @@ import pytest from arrow import Arrow from filelock import Timeout -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize import setup_configuration, start_hyperopt diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 21929de2b..ac4cbc813 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.resolvers import PairListResolver from freqtrade.pairlist.pairlistmanager import PairListManager diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index c5bb0ca2c..31632bd70 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -7,13 +7,13 @@ from unittest.mock import ANY, MagicMock, PropertyMock import pytest from numpy import isnan -from freqtrade import DependencyException, TemporaryError from freqtrade.edge import PairInfo +from freqtrade.exceptions import DependencyException, TemporaryError from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from tests.conftest import patch_get_signal, get_patched_freqtradebot +from tests.conftest import get_patched_freqtradebot, patch_get_signal # Functions for recurrent object patching diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 10b9f3466..d3977ae44 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -8,7 +8,7 @@ from pathlib import Path import pytest from pandas import DataFrame -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.interface import IStrategy from tests.conftest import log_has, log_has_re diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 6564417b3..ee3d23131 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -10,7 +10,6 @@ from unittest.mock import MagicMock import pytest from jsonschema import ValidationError -from freqtrade import OperationalException from freqtrade.configuration import (Arguments, Configuration, check_exchange, remove_credentials, validate_config_consistency) @@ -20,6 +19,7 @@ from freqtrade.configuration.deprecated_settings import ( process_temporary_deprecated_settings) from freqtrade.configuration.load_config import load_config_file from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL +from freqtrade.exceptions import OperationalException from freqtrade.loggers import _set_loggers, setup_logging from freqtrade.state import RunMode from tests.conftest import (log_has, log_has_re, diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py index db41e2da2..889338a64 100644 --- a/tests/test_directory_operations.py +++ b/tests/test_directory_operations.py @@ -4,10 +4,10 @@ from unittest.mock import MagicMock import pytest -from freqtrade import OperationalException from freqtrade.configuration.directory_operations import (copy_sample_files, create_datadir, create_userdata_dir) +from freqtrade.exceptions import OperationalException from tests.conftest import log_has, log_has_re diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2f8c786fd..3f5b7bd54 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,9 +11,9 @@ import arrow import pytest import requests -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError, constants) -from freqtrade.constants import MATH_CLOSE_PREC +from freqtrade.constants import MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType @@ -163,7 +163,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, ) conf = deepcopy(default_conf) - conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT conf['max_open_trades'] = 2 freqtrade = FreqtradeBot(conf) @@ -564,7 +564,7 @@ def test_create_trades_limit_reached(default_conf, ticker, limit_buy_order, get_fee=fee, ) default_conf['max_open_trades'] = 0 - default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + default_conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) diff --git a/tests/test_main.py b/tests/test_main.py index 03e6a7ce9..83be01999 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade import OperationalException from freqtrade.configuration import Arguments +from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main from freqtrade.state import State diff --git a/tests/test_persistence.py b/tests/test_persistence.py index b9a636e1a..6bd7971a7 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -6,7 +6,8 @@ import arrow import pytest from sqlalchemy import create_engine -from freqtrade import OperationalException, constants +from freqtrade import constants +from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade, clean_dry_run_db, init from tests.conftest import log_has diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 31502cafc..9934d2493 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -7,17 +7,17 @@ import plotly.graph_objects as go import pytest from plotly.subplots import make_subplots -from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data +from freqtrade.exceptions import OperationalException from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit from freqtrade.plot.plotting import (add_indicators, add_profit, - load_and_plot_trades, generate_candlestick_graph, generate_plot_filename, generate_profit_graph, init_plotscript, - plot_profit, plot_trades, store_plot_file) + load_and_plot_trades, plot_profit, + plot_trades, store_plot_file) from freqtrade.strategy.default_strategy import DefaultStrategy from tests.conftest import get_args, log_has, log_has_re diff --git a/tests/test_utils.py b/tests/test_utils.py index 4cf7b5f23..2e2499337 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from freqtrade.utils import (setup_utils_configuration, start_create_userdir, start_download_data, start_hyperopt_list, From 8e9a3e8fc8e5de762455ca7472c81e5ad15f8591 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Dec 2019 15:11:07 +0100 Subject: [PATCH 128/128] Capture FtBaseException at the outermost level --- freqtrade/exceptions.py | 28 +++++++++++++++++----------- freqtrade/main.py | 4 ++-- tests/test_main.py | 4 ++-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py index 80d45dd86..2f05ddb57 100644 --- a/freqtrade/exceptions.py +++ b/freqtrade/exceptions.py @@ -1,21 +1,27 @@ -class DependencyException(Exception): +class FreqtradeException(Exception): + """ + Freqtrade base exception. Handled at the outermost level. + All other exception types are subclasses of this exception type. + """ + + +class OperationalException(FreqtradeException): + """ + Requires manual intervention and will stop the bot. + Most of the time, this is caused by an invalid Configuration. + """ + + +class DependencyException(FreqtradeException): """ Indicates that an assumed dependency is not met. This could happen when there is currently not enough money on the account. """ -class OperationalException(Exception): - """ - Requires manual intervention and will usually stop the bot. - This happens when an exchange returns an unexpected error during runtime - or given configuration is invalid. - """ - - -class InvalidOrderException(Exception): +class InvalidOrderException(FreqtradeException): """ This is returned when the order is not valid. Example: If stoploss on exchange order is hit, then trying to cancel the order @@ -23,7 +29,7 @@ class InvalidOrderException(Exception): """ -class TemporaryError(Exception): +class TemporaryError(FreqtradeException): """ Temporary network or exchange related error. This could happen when an exchange is congested, unavailable, or the user diff --git a/freqtrade/main.py b/freqtrade/main.py index 62d2fc05d..811e29864 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -4,6 +4,7 @@ Main Freqtrade bot script. Read the documentation to know what cli arguments you need. """ +from freqtrade.exceptions import FreqtradeException, OperationalException import sys # check min. python version if sys.version_info < (3, 6): @@ -14,7 +15,6 @@ import logging from typing import Any, List from freqtrade.configuration import Arguments -from freqtrade.exceptions import OperationalException logger = logging.getLogger('freqtrade') @@ -50,7 +50,7 @@ def main(sysargv: List[str] = None) -> None: except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') return_code = 0 - except OperationalException as e: + except FreqtradeException as e: logger.error(str(e)) return_code = 2 except Exception: diff --git a/tests/test_main.py b/tests/test_main.py index 83be01999..76b1bf658 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock, PropertyMock import pytest from freqtrade.configuration import Arguments -from freqtrade.exceptions import OperationalException +from freqtrade.exceptions import OperationalException, FreqtradeException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main from freqtrade.state import State @@ -96,7 +96,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock()) mocker.patch( 'freqtrade.worker.Worker._worker', - MagicMock(side_effect=OperationalException('Oh snap!')) + MagicMock(side_effect=FreqtradeException('Oh snap!')) ) patched_configuration_load_config_file(mocker, default_conf) mocker.patch('freqtrade.wallets.Wallets.update', MagicMock())