From dec1b10743c3420a71b9dcc49d14c83d0dacb289 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 06:51:42 +0200 Subject: [PATCH 01/33] Add fee_cost and currency columns --- freqtrade/persistence.py | 22 +++++++++++++++++++--- tests/test_persistence.py | 4 ++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index fb314f439..36e2c7ffd 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -86,11 +86,15 @@ def check_migrate(engine) -> None: logger.debug(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'close_profit_abs'): + if not has_column(cols, 'fee_close_cost'): logger.info(f'Running database migration - backup available as {table_back_name}') fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') stop_loss = get_column_def(cols, 'stop_loss', '0.0') @@ -120,7 +124,9 @@ def check_migrate(engine) -> None: # Copy data back - following the correct schema engine.execute(f"""insert into trades - (id, exchange, pair, is_open, fee_open, fee_close, open_rate, + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, @@ -136,7 +142,9 @@ def check_migrate(engine) -> None: else pair end pair, - is_open, {fee_open} fee_open, {fee_close} fee_close, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, open_rate, {open_rate_requested} open_rate_requested, close_rate, {close_rate_requested} close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, @@ -185,7 +193,11 @@ class Trade(_DECL_BASE): pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) + fee_open_cost = Column(Float, nullable=True) + fee_open_currency = Column(String, nullable=True) fee_close = Column(Float, nullable=False, default=0.0) + fee_close_cost = Column(Float, nullable=True) + fee_close_currency = Column(String, nullable=True) open_rate = Column(Float) open_rate_requested = Column(Float) # open_trade_price - calculated via _calc_open_trade_price @@ -235,7 +247,11 @@ class Trade(_DECL_BASE): 'pair': self.pair, 'is_open': self.is_open, 'fee_open': self.fee_open, + 'fee_open_cost': self.fee_open_cost, + 'fee_open_currency': self.fee_open_currency, 'fee_close': self.fee_close, + 'fee_close_cost': self.fee_close_cost, + 'fee_close_currency': self.fee_close_currency, 'open_date_hum': arrow.get(self.open_date).humanize(), 'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"), 'close_date_hum': (arrow.get(self.close_date).humanize() diff --git a/tests/test_persistence.py b/tests/test_persistence.py index ceac24356..f54802ceb 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -465,6 +465,10 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.initial_stop_loss == 0.0 assert trade.open_trade_price == trade._calc_open_trade_price() assert trade.close_profit_abs is None + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None trade = Trade.query.filter(Trade.id == 2).first() assert trade.close_rate is not None From 7936120afc26104179fd1c8115e4f858fbc06b85 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 06:58:43 +0200 Subject: [PATCH 02/33] Adapt tests to support new db fields --- tests/rpc/test_rpc.py | 8 ++++++++ tests/rpc/test_rpc_apiserver.py | 8 ++++++++ tests/test_persistence.py | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index d2af4bd87..a1e6d9f26 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -51,7 +51,11 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_date_hum': ANY, 'is_open': ANY, 'fee_open': ANY, + 'fee_open_cost': ANY, + 'fee_open_currency': ANY, 'fee_close': ANY, + 'fee_close_cost': ANY, + 'fee_close_currency': ANY, 'open_rate_requested': ANY, 'open_trade_price': ANY, 'close_rate_requested': ANY, @@ -90,7 +94,11 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_date_hum': ANY, 'is_open': ANY, 'fee_open': ANY, + 'fee_open_cost': ANY, + 'fee_open_currency': ANY, 'fee_close': ANY, + 'fee_close_cost': ANY, + 'fee_close_currency': ANY, 'open_rate_requested': ANY, 'open_trade_price': ANY, 'close_rate_requested': ANY, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 6548790cb..01e9c0e96 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -472,7 +472,11 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'close_rate_requested': None, 'current_rate': 1.099e-05, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'open_date': ANY, 'is_open': True, 'max_rate': 0.0, @@ -575,7 +579,11 @@ def test_api_forcebuy(botclient, mocker, fee): 'close_profit': None, 'close_rate_requested': None, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'is_open': False, 'max_rate': None, 'min_rate': None, diff --git a/tests/test_persistence.py b/tests/test_persistence.py index f54802ceb..aa3a59b87 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -745,7 +745,11 @@ def test_to_json(default_conf, fee): 'open_rate_requested': None, 'open_trade_price': 15.1668225, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'close_rate': None, 'close_rate_requested': None, 'amount': 123.0, @@ -794,7 +798,11 @@ def test_to_json(default_conf, fee): 'close_profit': None, 'close_rate_requested': None, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'is_open': None, 'max_rate': None, 'min_rate': None, From b125dd3728b516f628363f5a7b7c8009a955c24e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 07:06:06 +0200 Subject: [PATCH 03/33] Extract order_has_fee method --- freqtrade/freqtradebot.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7ae87e807..80470f9e7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1162,6 +1162,17 @@ class FreqtradeBot: return False + def _order_has_fee(self, order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None and + (order['fee'].keys() >= {'currency', 'cost'})) + def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: """ Get real amount for the trade @@ -1175,8 +1186,7 @@ class FreqtradeBot: trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) # use fee from order-dict if possible - if ('fee' in order and order['fee'] is not None and - (order['fee'].keys() >= {'currency', 'cost'})): + if self._order_has_fee(order): if (order['fee']['currency'] is not None and order['fee']['cost'] is not None and trade_base_currency == order['fee']['currency']): @@ -1196,8 +1206,7 @@ class FreqtradeBot: fee_abs = 0 for exectrade in trades: amount += exectrade['amount'] - if ("fee" in exectrade and exectrade['fee'] is not None and - (exectrade['fee'].keys() >= {'currency', 'cost'})): + if self._order_has_fee(exectrade): # only applies if fee is in quote currency! if (exectrade['fee']['currency'] is not None and exectrade['fee']['cost'] is not None and From 6d7a3a0cc99d8e47b7e27e54d3827335af61a8bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 07:12:08 +0200 Subject: [PATCH 04/33] Extract more logic into order-has_fee --- freqtrade/freqtradebot.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 80470f9e7..4db9f9a1c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1164,14 +1164,18 @@ class FreqtradeBot: def _order_has_fee(self, order: Dict) -> bool: """ - Verifies if the passed in order dict has the needed keys to extract fees + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. :param order: Order or trade (one trade) dict :return: True if the fee substructure contains currency and cost, false otherwise """ if not isinstance(order, dict): return False - return ('fee' in order and order['fee'] is not None and - (order['fee'].keys() >= {'currency', 'cost'})) + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: """ @@ -1187,9 +1191,7 @@ class FreqtradeBot: trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) # use fee from order-dict if possible if self._order_has_fee(order): - if (order['fee']['currency'] is not None and - order['fee']['cost'] is not None and - trade_base_currency == order['fee']['currency']): + if trade_base_currency == order['fee']['currency']: new_amount = order_amount - order['fee']['cost'] logger.info("Applying fee on amount for %s (from %s to %s) from Order", trade, order['amount'], new_amount) @@ -1208,9 +1210,7 @@ class FreqtradeBot: amount += exectrade['amount'] if self._order_has_fee(exectrade): # only applies if fee is in quote currency! - if (exectrade['fee']['currency'] is not None and - exectrade['fee']['cost'] is not None and - trade_base_currency == exectrade['fee']['currency']): + if trade_base_currency == exectrade['fee']['currency']: fee_abs += exectrade['fee']['cost'] if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): From d3b9f4d393850de9d846735d8790d32dbc459c4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 19:28:36 +0200 Subject: [PATCH 05/33] Add extract_cost_curr_rate --- freqtrade/freqtradebot.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4db9f9a1c..7963fbec3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1177,6 +1177,15 @@ class FreqtradeBot: and order['fee']['cost'] is not None ) + def _extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, float]: + """ + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + order['fee'].get('rate', None)) + def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: """ Get real amount for the trade From a867d40eac3ab040eac09ac9905cefa4e74178fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 19:33:09 +0200 Subject: [PATCH 06/33] Move fee_methods to exchange --- freqtrade/exchange/exchange.py | 26 ++++++++++++++++++++++++++ freqtrade/freqtradebot.py | 24 ------------------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 1a0565959..9a0364b07 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1063,6 +1063,32 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + @staticmethod + def order_has_fee(order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) + + @staticmethod + def extract_cost_curr_rate(order: Dict) -> Tuple[float, str, float]: + """ + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + order['fee'].get('rate', None)) + def is_exchange_bad(exchange_name: str) -> bool: return exchange_name in BAD_EXCHANGES diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7963fbec3..a297f1823 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1162,30 +1162,6 @@ class FreqtradeBot: return False - def _order_has_fee(self, order: Dict) -> bool: - """ - Verifies if the passed in order dict has the needed keys to extract fees, - and that these keys (currency, cost) are not empty. - :param order: Order or trade (one trade) dict - :return: True if the fee substructure contains currency and cost, false otherwise - """ - if not isinstance(order, dict): - return False - return ('fee' in order and order['fee'] is not None - and (order['fee'].keys() >= {'currency', 'cost'}) - and order['fee']['currency'] is not None - and order['fee']['cost'] is not None - ) - - def _extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, float]: - """ - :param order: Order or trade (one trade) dict - :return: Tuple with cost, currency, rate of the given fee dict - """ - return (order['fee']['cost'], - order['fee']['currency'], - order['fee'].get('rate', None)) - def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: """ Get real amount for the trade From 2e4dc6c25388feaa8ce13407bf62d2eabfb27938 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 19:56:48 +0200 Subject: [PATCH 07/33] Exchange should return fee dict for dry-run orders --- freqtrade/exchange/exchange.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 9a0364b07..22d477e6c 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -472,26 +472,31 @@ class Exchange: 'pair': pair, 'price': rate, 'amount': _amount, - "cost": _amount * rate, + 'cost': _amount * rate, 'type': ordertype, 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), 'status': "closed" if ordertype == "market" else "open", 'fee': None, - "info": {} + 'info': {} } - self._store_dry_order(dry_order) + self._store_dry_order(dry_order, pair) # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def _store_dry_order(self, dry_order: Dict) -> None: + def _store_dry_order(self, dry_order: Dict, pair: str) -> None: closed_order = dry_order.copy() - if closed_order["type"] in ["market", "limit"]: + if closed_order['type'] in ["market", "limit"]: closed_order.update({ - "status": "closed", - "filled": closed_order["amount"], - "remaining": 0 + 'status': 'closed', + 'filled': closed_order['amount'], + 'remaining': 0, + 'fee': { + 'currency': self.get_pair_quote_currency(pair), + 'cost': dry_order['amount'] * self.get_fee(pair), + 'rate': self.get_fee(pair) + } }) if closed_order["type"] in ["stop_loss_limit"]: closed_order["info"].update({"stopPrice": closed_order["price"]}) From e74ed0ba7b7ba4f9f8f6cd8f456d1ccc35901614 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 20:05:27 +0200 Subject: [PATCH 08/33] Add tests for fee extraction methods --- freqtrade/exchange/exchange.py | 2 ++ tests/exchange/test_exchange.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 22d477e6c..bd6d617c0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1087,6 +1087,8 @@ class Exchange: @staticmethod def extract_cost_curr_rate(order: Dict) -> Tuple[float, str, float]: """ + Extract tuple of cost, currency, rate. + Requires order_has_fee to run first! :param order: Order or trade (one trade) dict :return: Tuple with cost, currency, rate of the given fee dict """ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 3c92612a0..f8c572dc8 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2145,3 +2145,23 @@ def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_r ]) def test_market_is_active(market, expected_result) -> None: assert market_is_active(market) == expected_result + + +@pytest.mark.parametrize("order,expected", [ + ([{'fee'}], False), + ({'fee': None}, False), + ({'fee': {'currency': 'ETH/BTC'}}, False), + ({'fee': {'currency': 'ETH/BTC', 'cost': None}}, False), + ({'fee': {'currency': 'ETH/BTC', 'cost': 0.01}}, True), +]) +def test_order_has_fee(order, expected) -> None: + assert Exchange.order_has_fee(order) == expected + + +@pytest.mark.parametrize("order,expected", [ + ({'fee': {'currency': 'ETH/BTC', 'cost': 0.43}}, (0.43, 'ETH/BTC', None)), + ({'fee': {'currency': 'ETH/USDT', 'cost': 0.01}}, (0.01, 'ETH/USDT', None)), + ({'fee': {'currency': 'ETH/USDT', 'cost': 0.34, 'rate': 0.01}}, (0.34, 'ETH/USDT', 0.01)), +]) +def test_extract_cost_curr_rate(order, expected) -> None: + assert Exchange.extract_cost_curr_rate(order) == expected From f8f794a8038ec9d532b39cf7a947956e02b7bac0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 20:15:35 +0200 Subject: [PATCH 09/33] Simplify fee-related tests --- tests/test_freqtradebot.py | 60 ++++++++------------------------------ 1 file changed, 12 insertions(+), 48 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 6f2ce9f3c..43ee4c79d 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3285,8 +3285,6 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - patch_RPCManager(mocker) - patch_exchange(mocker) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -3297,8 +3295,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) @@ -3310,8 +3307,6 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) - patch_RPCManager(mocker) - patch_exchange(mocker) amount = buy_order_fee['amount'] trade = Trade( pair='LTC/ETH', @@ -3322,8 +3317,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -3335,8 +3329,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): trades_for_order[0]['fee']['currency'] = 'ETH' - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3348,8 +3340,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -3362,8 +3353,6 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ limit_buy_order['fee'] = {'cost': 0.004, 'currency': None} trades_for_order[0]['fee']['currency'] = None - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3375,8 +3364,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -3386,8 +3374,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, trades_for_order[0]['fee']['currency'] = 'BNB' trades_for_order[0]['fee']['cost'] = 0.00094518 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3399,16 +3385,13 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) amount = float(sum(x['amount'] for x in trades_for_order2)) trade = Trade( @@ -3420,8 +3403,7 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) @@ -3435,8 +3417,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[trades_for_order]) amount = float(sum(x['amount'] for x in trades_for_order)) @@ -3449,8 +3429,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 @@ -3463,8 +3442,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3476,8 +3453,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -3487,8 +3463,6 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['amount'] = limit_buy_order['amount'] - 0.001 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3500,8 +3474,7 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change with pytest.raises(DependencyException, match=r"Half bought\? Amounts don't match"): @@ -3514,8 +3487,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b limit_buy_order = deepcopy(buy_order_fee) trades_for_order[0]['amount'] = trades_for_order[0]['amount'] + 1e-15 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3527,8 +3498,7 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount changes by fee amount. assert isclose(freqtrade.get_real_amount(trade, limit_buy_order), amount - (amount * 0.001), @@ -3539,8 +3509,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, # Remove "Currency" from fee dict trades_for_order[0]['fee'] = {'cost': 0.008} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3553,15 +3521,12 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount def test_get_real_amount_open_trade(default_conf, fee, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) amount = 12345 trade = Trade( pair='LTC/ETH', @@ -3577,8 +3542,7 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): 'amount': amount, 'status': 'open', } - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.get_real_amount(trade, order) == amount From 45c97fde2dc125f00255f197f5bbf4fe63a20a06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Apr 2020 22:16:52 +0200 Subject: [PATCH 10/33] Use correct typehint for extract_cost_curr_rate --- freqtrade/exchange/exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index bd6d617c0..df7f3005e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1085,7 +1085,7 @@ class Exchange: ) @staticmethod - def extract_cost_curr_rate(order: Dict) -> Tuple[float, str, float]: + def extract_cost_curr_rate(order: Dict) -> Tuple[float, str, Optional[float]]: """ Extract tuple of cost, currency, rate. Requires order_has_fee to run first! @@ -1095,6 +1095,7 @@ class Exchange: return (order['fee']['cost'], order['fee']['currency'], order['fee'].get('rate', None)) + # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price'])) def is_exchange_bad(exchange_name: str) -> bool: From 59bafc8d02e291bb0797eeecdc1cd92feadb5517 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 15:17:52 +0200 Subject: [PATCH 11/33] Implement fee rate calculation --- freqtrade/exchange/exchange.py | 33 +++++++++++++++++++++--- tests/exchange/test_exchange.py | 45 +++++++++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index df7f3005e..ebadd4c8a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -22,7 +22,7 @@ from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.exceptions import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, safe_value_fallback CcxtModuleType = Any @@ -1084,8 +1084,33 @@ class Exchange: and order['fee']['cost'] is not None ) - @staticmethod - def extract_cost_curr_rate(order: Dict) -> Tuple[float, str, Optional[float]]: + def calculate_fee_rate(self, order: Dict) -> Optional[float]: + """ + Calculate fee rate if it's not given by the exchange. + :param order: Order or trade (one trade) dict + """ + if order['fee'].get('rate') is not None: + return order['fee'].get('rate') + fee_curr = order['fee']['currency'] + # Calculate fee based on order details + if fee_curr in self.get_pair_base_currency(order['symbol']): + # Base currency - divide by amount + return round(order['fee']['cost'] / order['amount'], 8) + elif fee_curr in self.get_pair_quote_currency(order['symbol']): + # Quote currency - divide by cost + return round(order['fee']['cost'] / order['cost'], 8) + else: + # If Fee currency is a different currency + try: + comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) + tick = self.fetch_ticker(comb) + + fee_to_quote_rate = safe_value_fallback(tick, tick, 'last', 'ask') + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) + except DependencyException: + return None + + def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: """ Extract tuple of cost, currency, rate. Requires order_has_fee to run first! @@ -1094,7 +1119,7 @@ class Exchange: """ return (order['fee']['cost'], order['fee']['currency'], - order['fee'].get('rate', None)) + self.calculate_fee_rate(order)) # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price'])) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f8c572dc8..7a1df5103 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2159,9 +2159,44 @@ def test_order_has_fee(order, expected) -> None: @pytest.mark.parametrize("order,expected", [ - ({'fee': {'currency': 'ETH/BTC', 'cost': 0.43}}, (0.43, 'ETH/BTC', None)), - ({'fee': {'currency': 'ETH/USDT', 'cost': 0.01}}, (0.01, 'ETH/USDT', None)), - ({'fee': {'currency': 'ETH/USDT', 'cost': 0.34, 'rate': 0.01}}, (0.34, 'ETH/USDT', 0.01)), + ({'symbol': 'ETH/BTC', 'fee': {'currency': 'ETH', 'cost': 0.43}}, + (0.43, 'ETH', 0.01)), + ({'symbol': 'ETH/USDT', 'fee': {'currency': 'USDT', 'cost': 0.01}}, + (0.01, 'USDT', 0.01)), + ({'symbol': 'BTC/USDT', 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, + (0.34, 'USDT', 0.01)), ]) -def test_extract_cost_curr_rate(order, expected) -> None: - assert Exchange.extract_cost_curr_rate(order) == expected +def test_extract_cost_curr_rate(mocker, default_conf, order, expected) -> None: + mocker.patch('freqtrade.exchange.Exchange.calculate_fee_rate', MagicMock(return_value=0.01)) + ex = get_patched_exchange(mocker, default_conf) + assert ex.extract_cost_curr_rate(order) == expected + + +@pytest.mark.parametrize("order,expected", [ + # Using base-currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.1), + ({'symbol': 'ETH/BTC', 'amount': 0.05, 'cost': 0.05, + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.08), + # Using quote currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'BTC', 'cost': 0.005}}, 0.1), + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'BTC', 'cost': 0.002, 'rate': None}}, 0.04), + # Using foreign currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'NEO', 'cost': 0.0012}}, 0.001944), + ({'symbol': 'ETH/BTC', 'amount': 2.21, 'cost': 0.02992561, + 'fee': {'currency': 'NEO', 'cost': 0.00027452}}, 0.00074305), + # TODO: More tests here! + # Rate included in return - return as is + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, 0.01), + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.005}}, 0.005), +]) +def test_calculate_fee_rate(mocker, default_conf, order, expected) -> None: + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'last': 0.081}) + + ex = get_patched_exchange(mocker, default_conf) + assert ex.calculate_fee_rate(order) == expected From db8fb39a3860caca6000f384055d5476ba60f03b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 15:20:38 +0200 Subject: [PATCH 12/33] don't use trade.update directly! --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a297f1823..ff2d1d759 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1032,7 +1032,7 @@ class FreqtradeBot: trade.sell_reason = sell_reason.value # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': - trade.update(order) + self.update_trade_state(trade, order) Trade.session.flush() # Lock pair for one candle to prevent immediate rebuys From b93d33a93a652c3c870d43a1ca75b4c7aadbe39b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 15:30:49 +0200 Subject: [PATCH 13/33] Fix mock order dicts --- tests/conftest.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d95475b8c..621f45407 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -779,7 +779,7 @@ def limit_buy_order(): 'id': 'mocked_limit_buy', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00001099, 'amount': 90.99181073, @@ -795,7 +795,7 @@ def market_buy_order(): 'id': 'mocked_market_buy', 'type': 'market', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00004099, 'amount': 91.99181073, @@ -811,7 +811,7 @@ def market_sell_order(): 'id': 'mocked_limit_sell', 'type': 'market', 'side': 'sell', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00004173, 'amount': 91.99181073, @@ -827,7 +827,7 @@ def limit_buy_order_old(): 'id': 'mocked_limit_buy_old', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': str(arrow.utcnow().shift(minutes=-601).datetime), 'price': 0.00001099, 'amount': 90.99181073, @@ -843,7 +843,7 @@ def limit_sell_order_old(): 'id': 'mocked_limit_sell_old', 'type': 'limit', 'side': 'sell', - 'pair': 'ETH/BTC', + 'symbol': 'ETH/BTC', 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, @@ -859,7 +859,7 @@ def limit_buy_order_old_partial(): 'id': 'mocked_limit_buy_old_partial', 'type': 'limit', 'side': 'buy', - 'pair': 'ETH/BTC', + 'symbol': 'ETH/BTC', 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, @@ -1482,7 +1482,7 @@ def buy_order_fee(): 'id': 'mocked_limit_buy_old', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': str(arrow.utcnow().shift(minutes=-601).datetime), 'price': 0.245441, 'amount': 8.0, From fdcc507f06ff725a7c0053676509ba049c7e8129 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 15:35:57 +0200 Subject: [PATCH 14/33] Fix integration tests --- tests/test_integration.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index c40da7e9d..ee6ef3cb2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -67,7 +67,6 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - update_trade_state=MagicMock(), _notify_sell=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) @@ -97,8 +96,9 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, # Only order for 3rd trade needs to be cancelled assert cancel_order_mock.call_count == 1 - # Wallets must be updated between stoploss cancellation and selling. - assert wallets_mock.call_count == 2 + # Wallets must be updated between stoploss cancellation and selling, and will be updated again + # During update_trade_state + assert wallets_mock.call_count == 3 trade = trades[0] assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value @@ -144,7 +144,6 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - update_trade_state=MagicMock(), _notify_sell=MagicMock(), ) should_sell_mock = MagicMock(side_effect=[ From a2ff6326470afde76cdfe9d70966d83151afbded Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 16:00:42 +0200 Subject: [PATCH 15/33] Add update_fee method to persistence --- freqtrade/persistence.py | 18 ++++++++++++++++++ tests/test_persistence.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 36e2c7ffd..fec095daa 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -376,6 +376,24 @@ class Trade(_DECL_BASE): self ) + def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], + side: str) -> None: + """ + Update Fee parameters. Only acts once per side + """ + if side == 'buy' and self.fee_open_currency is None: + self.fee_open_cost = fee_cost + self.fee_open_currency = fee_currency + if fee_rate is not None: + self.fee_open = fee_rate + # Assume close-fee will fall into the same fee category and take an educated guess + self.fee_close = fee_rate + elif side == 'sell' and self.fee_close_currency is None: + self.fee_close_cost = fee_cost + self.fee_close_currency = fee_currency + if fee_rate is not None: + self.fee_close = fee_rate + def _calc_open_trade_price(self) -> float: """ Calculate the open_rate including open_fee. diff --git a/tests/test_persistence.py b/tests/test_persistence.py index aa3a59b87..fa6f84ce0 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -874,6 +874,39 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade_adj.initial_stop_loss_pct == -0.04 +def test_update_fee(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + amount=10, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + max_rate=1, + ) + fee_cost = 0.15 + fee_currency = 'BTC' + fee_rate = 0.0075 + assert trade.fee_open_currency is None + + trade.update_fee(fee_cost, fee_currency, fee_rate, 'buy') + assert trade.fee_open_currency == fee_currency + assert trade.fee_open_cost == fee_cost + assert trade.fee_open == fee_rate + # Setting buy rate should "guess" close rate + assert trade.fee_close == fee_rate + assert trade.fee_close_currency is None + assert trade.fee_close_cost is None + + fee_rate = 0.0076 + trade.update_fee(fee_cost, fee_currency, fee_rate, 'sell') + assert trade.fee_close == 0.0076 + assert trade.fee_close_cost == fee_cost + assert trade.fee_close == fee_rate + + @pytest.mark.usefixtures("init_persistence") def test_total_open_trades_stakes(fee): From 371100a97c049d4090358b97c36d4372ebfe8616 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 16:01:33 +0200 Subject: [PATCH 16/33] Update fee handling --- freqtrade/freqtradebot.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ff2d1d759..063712ed8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1167,20 +1167,25 @@ class FreqtradeBot: Get real amount for the trade Necessary for exchanges which charge fees in base currency (e.g. binance) """ + fee_currency = None + # Init variables if order_amount is None: order_amount = order['amount'] # Only run for closed orders - if trade.fee_open == 0 or order['status'] == 'open': + if trade.fee_open_currency is not None or order['status'] == 'open': return order_amount trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) # use fee from order-dict if possible - if self._order_has_fee(order): - if trade_base_currency == order['fee']['currency']: - new_amount = order_amount - order['fee']['cost'] - logger.info("Applying fee on amount for %s (from %s to %s) from Order", - trade, order['amount'], new_amount) - return new_amount + if self.exchange.order_has_fee(order): + fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) + logger.info(f"Fee for Trade {trade}: {fee_cost:.8g} {fee_currency} - rate: {fee_rate}") + if trade_base_currency == fee_currency: + order_amount = order_amount - fee_cost + logger.info(f"Applying fee on amount for {trade} (from {order['amount']} " + f"to {order_amount}) from Order") + trade.update_fee(fee_cost, fee_currency, fee_rate, order['side']) + return order_amount # Fallback to Trades trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, @@ -1190,13 +1195,25 @@ class FreqtradeBot: logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) return order_amount amount = 0 - fee_abs = 0 + fee_abs = 0.0 + fee_cost = 0.0 + fee_rate_array: List[float] = [] for exectrade in trades: amount += exectrade['amount'] - if self._order_has_fee(exectrade): + if self.exchange.order_has_fee(exectrade): + fee_cost_, fee_currency, fee_rate_ = self.exchange.extract_cost_curr_rate(exectrade) + fee_cost += fee_cost_ + if fee_rate_ is not None: + fee_rate_array.append(fee_rate_) # only applies if fee is in quote currency! - if trade_base_currency == exectrade['fee']['currency']: - fee_abs += exectrade['fee']['cost'] + if trade_base_currency == fee_currency: + fee_abs += fee_cost_ + # Ensure at least one trade was found: + if fee_currency: + # fee_rate should use mean + + fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None + trade.update_fee(fee_cost, fee_currency, fee_rate, order['side']) if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): logger.warning(f"Amount {amount} does not match amount {trade.amount}") From 7558e4bffe6fdcb5a2883f16d5b817defb56f59f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 16:13:07 +0200 Subject: [PATCH 17/33] Split fee detection from trades from general logic --- freqtrade/freqtradebot.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 063712ed8..ed228f6ad 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1164,10 +1164,12 @@ class FreqtradeBot: def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: """ - Get real amount for the trade + Detect and update trade fee. + Calls trade.update_fee() uppon correct detection. + Returns modified amount if the fee was taken from the destination currency. Necessary for exchanges which charge fees in base currency (e.g. binance) + :return: identical (or new) amount for the trade """ - fee_currency = None # Init variables if order_amount is None: order_amount = order['amount'] @@ -1186,17 +1188,23 @@ class FreqtradeBot: f"to {order_amount}) from Order") trade.update_fee(fee_cost, fee_currency, fee_rate, order['side']) return order_amount + return self.fee_detection_from_trades(trade, order, order_amount) - # Fallback to Trades + def fee_detection_from_trades(self, trade: Trade, order: Dict, order_amount: float) -> float: + """ + fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. + """ trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, trade.open_date) if len(trades) == 0: logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) return order_amount + fee_currency = None amount = 0 fee_abs = 0.0 fee_cost = 0.0 + trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) fee_rate_array: List[float] = [] for exectrade in trades: amount += exectrade['amount'] From 40d4949f0669423c89011a98b804f41a437815d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 17:46:01 +0200 Subject: [PATCH 18/33] Don't handle trades if they've just been closed. --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ed228f6ad..72531c9f9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -610,7 +610,7 @@ class FreqtradeBot: trades_closed += 1 continue # Check if we can sell our current pair - if trade.open_order_id is None and self.handle_trade(trade): + if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): trades_closed += 1 except DependencyException as exception: From 6b33d5af1e440253019a125551423f52fa26d7d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 19:51:35 +0200 Subject: [PATCH 19/33] Fix fee-calculation for dry-run orders --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ebadd4c8a..b7547cacb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -494,7 +494,7 @@ class Exchange: 'remaining': 0, 'fee': { 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['amount'] * self.get_fee(pair), + 'cost': dry_order['cost'] * self.get_fee(pair), 'rate': self.get_fee(pair) } }) From 6d620ba1f6946b5b79a95ec3cc1c3c63ec8f5a1f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 19:54:16 +0200 Subject: [PATCH 20/33] Verify if fee for this side has been updated --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence.py | 9 +++++++++ tests/test_persistence.py | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 72531c9f9..a3dd29771 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1174,7 +1174,7 @@ class FreqtradeBot: if order_amount is None: order_amount = order['amount'] # Only run for closed orders - if trade.fee_open_currency is not None or order['status'] == 'open': + if trade.fee_updated(order['side']) or order['status'] == 'open': return order_amount trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index fec095daa..a541dcbcc 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -394,6 +394,15 @@ class Trade(_DECL_BASE): if fee_rate is not None: self.fee_close = fee_rate + def fee_updated(self, side: str) -> bool: + """ + Verify if this side (buy / sell) has already been updated + """ + if side == 'buy': + return self.fee_open_currency is not None + elif side == 'sell': + return self.fee_close_currency is not None + def _calc_open_trade_price(self) -> float: """ Calculate the open_rate including open_fee. diff --git a/tests/test_persistence.py b/tests/test_persistence.py index fa6f84ce0..1a42d95fe 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -890,8 +890,12 @@ def test_update_fee(fee): fee_currency = 'BTC' fee_rate = 0.0075 assert trade.fee_open_currency is None + assert not trade.fee_updated('buy') + assert not trade.fee_updated('sell') trade.update_fee(fee_cost, fee_currency, fee_rate, 'buy') + assert trade.fee_updated('buy') + assert not trade.fee_updated('sell') assert trade.fee_open_currency == fee_currency assert trade.fee_open_cost == fee_cost assert trade.fee_open == fee_rate @@ -902,6 +906,8 @@ def test_update_fee(fee): fee_rate = 0.0076 trade.update_fee(fee_cost, fee_currency, fee_rate, 'sell') + assert trade.fee_updated('buy') + assert trade.fee_updated('sell') assert trade.fee_close == 0.0076 assert trade.fee_close_cost == fee_cost assert trade.fee_close == fee_rate From 71c90422ba5f4c34b8689864d35c92cee0c5e8d5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 20:02:38 +0200 Subject: [PATCH 21/33] Add explicit test for fee_updated --- freqtrade/persistence.py | 2 ++ tests/test_persistence.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index a541dcbcc..ea34fd5bf 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -402,6 +402,8 @@ class Trade(_DECL_BASE): return self.fee_open_currency is not None elif side == 'sell': return self.fee_close_currency is not None + else: + return False def _calc_open_trade_price(self) -> float: """ diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 1a42d95fe..5c7686e28 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -913,6 +913,36 @@ def test_update_fee(fee): assert trade.fee_close == fee_rate +def test_fee_updated(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + amount=10, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + max_rate=1, + ) + + assert trade.fee_open_currency is None + assert not trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert not trade.fee_updated('asdf') + + trade.update_fee(0.15, 'BTC', 0.0075, 'buy') + assert trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert trade.fee_open_currency is not None + assert trade.fee_close_currency is None + + trade.update_fee(0.15, 'ABC', 0.0075, 'sell') + assert trade.fee_updated('buy') + assert trade.fee_updated('sell') + assert not trade.fee_updated('asfd') + + @pytest.mark.usefixtures("init_persistence") def test_total_open_trades_stakes(fee): From 737fc6d19837c816504c7dd910e9ec702fe1bc57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 20:03:06 +0200 Subject: [PATCH 22/33] Fix bug when querying side --- freqtrade/freqtradebot.py | 9 +++++---- freqtrade/persistence.py | 2 +- tests/test_freqtradebot.py | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a3dd29771..d0a9a8348 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1174,19 +1174,20 @@ class FreqtradeBot: if order_amount is None: order_amount = order['amount'] # Only run for closed orders - if trade.fee_updated(order['side']) or order['status'] == 'open': + if trade.fee_updated(order.get('side')) or order['status'] == 'open': return order_amount trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) # use fee from order-dict if possible if self.exchange.order_has_fee(order): fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) - logger.info(f"Fee for Trade {trade}: {fee_cost:.8g} {fee_currency} - rate: {fee_rate}") + logger.info(f"Fee for Trade {trade} [{order.get('side')}]: " + f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}") if trade_base_currency == fee_currency: order_amount = order_amount - fee_cost logger.info(f"Applying fee on amount for {trade} (from {order['amount']} " f"to {order_amount}) from Order") - trade.update_fee(fee_cost, fee_currency, fee_rate, order['side']) + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side')) return order_amount return self.fee_detection_from_trades(trade, order, order_amount) @@ -1221,7 +1222,7 @@ class FreqtradeBot: # fee_rate should use mean fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None - trade.update_fee(fee_cost, fee_currency, fee_rate, order['side']) + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side')) if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): logger.warning(f"Amount {amount} does not match amount {trade.amount}") diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ea34fd5bf..0f2c6bb52 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -394,7 +394,7 @@ class Trade(_DECL_BASE): if fee_rate is not None: self.fee_close = fee_rate - def fee_updated(self, side: str) -> bool: + def fee_updated(self, side) -> bool: """ Verify if this side (buy / sell) has already been updated """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 43ee4c79d..cf5d30a79 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3541,6 +3541,7 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): 'id': 'mocked_order', 'amount': amount, 'status': 'open', + 'side': 'buy', } freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.get_real_amount(trade, order) == amount From 021e2b58ca87babab4d18f574cdc0ca435e1639d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 20:17:22 +0200 Subject: [PATCH 23/33] Support partially cancelled orders for fee calculation --- freqtrade/exchange/exchange.py | 3 ++- freqtrade/freqtradebot.py | 2 -- freqtrade/persistence.py | 2 +- tests/conftest.py | 2 +- tests/test_freqtradebot.py | 5 +++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b7547cacb..e649e5ed2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1095,7 +1095,8 @@ class Exchange: # Calculate fee based on order details if fee_curr in self.get_pair_base_currency(order['symbol']): # Base currency - divide by amount - return round(order['fee']['cost'] / order['amount'], 8) + return round( + order['fee']['cost'] / safe_value_fallback(order, order, 'filled', 'amount'), 8) elif fee_curr in self.get_pair_quote_currency(order['symbol']): # Quote currency - divide by cost return round(order['fee']['cost'] / order['cost'], 8) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d0a9a8348..32adc6d57 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1144,8 +1144,6 @@ class FreqtradeBot: if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): order['amount'] = new_amount order.pop('filled', None) - # Fee was applied, so set to 0 - trade.fee_open = 0 trade.recalc_open_trade_price() except DependencyException as exception: logger.warning("Could not update trade amount: %s", exception) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 0f2c6bb52..17246633c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -377,7 +377,7 @@ class Trade(_DECL_BASE): ) def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], - side: str) -> None: + side) -> None: """ Update Fee parameters. Only acts once per side """ diff --git a/tests/conftest.py b/tests/conftest.py index 621f45407..10885d667 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -873,7 +873,7 @@ def limit_buy_order_old_partial(): def limit_buy_order_old_partial_canceled(limit_buy_order_old_partial): res = deepcopy(limit_buy_order_old_partial) res['status'] = 'canceled' - res['fee'] = {'cost': 0.0001, 'currency': 'ETH'} + res['fee'] = {'cost': 0.023, 'currency': 'ETH'} return res diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index cf5d30a79..19a094fab 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2229,9 +2229,10 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert len(trades) == 1 # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - - limit_buy_order_old_partial['remaining']) - 0.0001 + limit_buy_order_old_partial['remaining']) - 0.023 assert trades[0].open_order_id is None - assert trades[0].fee_open == 0 + assert trades[0].fee_updated('buy') + assert pytest.approx(trades[0].fee_open) == 0.001 def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, From 78b3d8487f8509c602818f1b241e524c5f24feac Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 May 2020 20:34:58 +0200 Subject: [PATCH 24/33] Add typehint --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/persistence.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 32adc6d57..163c4faaa 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1172,7 +1172,7 @@ class FreqtradeBot: if order_amount is None: order_amount = order['amount'] # Only run for closed orders - if trade.fee_updated(order.get('side')) or order['status'] == 'open': + if trade.fee_updated(order.get('side', '')) or order['status'] == 'open': return order_amount trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) @@ -1185,7 +1185,7 @@ class FreqtradeBot: order_amount = order_amount - fee_cost logger.info(f"Applying fee on amount for {trade} (from {order['amount']} " f"to {order_amount}) from Order") - trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side')) + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) return order_amount return self.fee_detection_from_trades(trade, order, order_amount) @@ -1220,7 +1220,7 @@ class FreqtradeBot: # fee_rate should use mean fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None - trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side')) + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): logger.warning(f"Amount {amount} does not match amount {trade.amount}") diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 17246633c..ea34fd5bf 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -377,7 +377,7 @@ class Trade(_DECL_BASE): ) def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], - side) -> None: + side: str) -> None: """ Update Fee parameters. Only acts once per side """ @@ -394,7 +394,7 @@ class Trade(_DECL_BASE): if fee_rate is not None: self.fee_close = fee_rate - def fee_updated(self, side) -> bool: + def fee_updated(self, side: str) -> bool: """ Verify if this side (buy / sell) has already been updated """ From 38c49493606e409aae7b6f8a081e06cca1414e51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 May 2020 10:50:59 +0200 Subject: [PATCH 25/33] Align applying of fee when comming from orders or trades --- freqtrade/freqtradebot.py | 36 +++++++++++++++++++++++++++--------- tests/test_freqtradebot.py | 8 ++++---- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 163c4faaa..5ef4888e0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1160,6 +1160,23 @@ class FreqtradeBot: return False + def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, + amount: float, fee: float) -> float: + """ + Applies the fee to amount (either from Order or from Trades). + Can eat into dust if more than the required asset is available. + """ + if fee != 0 and self.wallets.get_free(trade_base_currency) >= amount: + # Eat into dust if we own more than base currency + logger.info(f"Fee amount for {trade} was in base currency - " + f"Eating Fee {fee} into dust.") + elif fee != 0: + real_amount = self.exchange.amount_to_precision(trade.pair, amount - fee) + logger.info(f"Applying fee on amount for {trade} " + f"(from {amount} to {real_amount}).") + return real_amount + return amount + def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: """ Detect and update trade fee. @@ -1181,11 +1198,12 @@ class FreqtradeBot: fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) logger.info(f"Fee for Trade {trade} [{order.get('side')}]: " f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}") - if trade_base_currency == fee_currency: - order_amount = order_amount - fee_cost - logger.info(f"Applying fee on amount for {trade} (from {order['amount']} " - f"to {order_amount}) from Order") + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + if trade_base_currency == fee_currency: + # Apply fee to amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=order_amount, fee=fee_cost) return order_amount return self.fee_detection_from_trades(trade, order, order_amount) @@ -1218,15 +1236,15 @@ class FreqtradeBot: # Ensure at least one trade was found: if fee_currency: # fee_rate should use mean - fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): logger.warning(f"Amount {amount} does not match amount {trade.amount}") raise DependencyException("Half bought? Amounts don't match") - real_amount = amount - fee_abs + if fee_abs != 0: - logger.info(f"Applying fee on amount for {trade} " - f"(from {order_amount} to {real_amount}) from Trades") - return real_amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=amount, fee=fee_abs) + else: + return amount diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 19a094fab..4a28147fc 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2221,7 +2221,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap # and apply fees if necessary. freqtrade.check_handle_timedout() - assert log_has_re(r"Applying fee on amount for Trade.* Order", caplog) + assert log_has_re(r"Applying fee on amount for Trade.*", caplog) assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 @@ -3301,7 +3301,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) @@ -3409,7 +3409,7 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) @@ -3435,7 +3435,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996) from Order', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', caplog) From 58168336e1801ba061c5498c007380759756f89d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 May 2020 11:13:59 +0200 Subject: [PATCH 26/33] Add test for apply_fee_conditional --- freqtrade/freqtradebot.py | 15 ++++++++------- tests/test_freqtradebot.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5ef4888e0..1f9573122 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1161,17 +1161,18 @@ class FreqtradeBot: return False def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, - amount: float, fee: float) -> float: + amount: float, fee_abs: float) -> float: """ Applies the fee to amount (either from Order or from Trades). Can eat into dust if more than the required asset is available. """ - if fee != 0 and self.wallets.get_free(trade_base_currency) >= amount: + self.wallets.update() + if fee_abs != 0 and self.wallets.get_free(trade_base_currency) >= amount: # Eat into dust if we own more than base currency logger.info(f"Fee amount for {trade} was in base currency - " - f"Eating Fee {fee} into dust.") - elif fee != 0: - real_amount = self.exchange.amount_to_precision(trade.pair, amount - fee) + f"Eating Fee {fee_abs} into dust.") + elif fee_abs != 0: + real_amount = self.exchange.amount_to_precision(trade.pair, amount - fee_abs) logger.info(f"Applying fee on amount for {trade} " f"(from {amount} to {real_amount}).") return real_amount @@ -1203,7 +1204,7 @@ class FreqtradeBot: if trade_base_currency == fee_currency: # Apply fee to amount return self.apply_fee_conditional(trade, trade_base_currency, - amount=order_amount, fee=fee_cost) + amount=order_amount, fee_abs=fee_cost) return order_amount return self.fee_detection_from_trades(trade, order, order_amount) @@ -1245,6 +1246,6 @@ class FreqtradeBot: if fee_abs != 0: return self.apply_fee_conditional(trade, trade_base_currency, - amount=amount, fee=fee_abs) + amount=amount, fee_abs=fee_abs) else: return amount diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 4a28147fc..e69c88a13 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3305,6 +3305,30 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe caplog) +def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) + walletmock = mocker.patch('freqtrade.wallets.Wallets.update') + mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=8.1122) + amount = sum(x['amount'] for x in trades_for_order) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_order_id="123456" + ) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + walletmock.reset_mock() + # Amount is kept as is + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + assert walletmock.call_count == 1 + assert log_has_re(r'Fee amount for Trade.* was in base currency ' + '- Eating Fee 0.008 into dust', caplog) + + def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) From 72282a22396b59ee92c4e6a3c01dd365dafac9e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 May 2020 11:28:29 +0200 Subject: [PATCH 27/33] Add explicit test for fee_conditional --- tests/test_freqtradebot.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e69c88a13..2c97b9fe1 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3305,7 +3305,8 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe caplog) -def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): +def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, + caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) walletmock = mocker.patch('freqtrade.wallets.Wallets.update') mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=8.1122) @@ -3326,7 +3327,7 @@ def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fe assert freqtrade.get_real_amount(trade, buy_order_fee) == amount assert walletmock.call_count == 1 assert log_has_re(r'Fee amount for Trade.* was in base currency ' - '- Eating Fee 0.008 into dust', caplog) + '- Eating Fee 0.008 into dust', caplog) def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): @@ -3572,6 +3573,36 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): assert freqtrade.get_real_amount(trade, order) == amount +@pytest.mark.parametrize('amount,fee_abs,wallet,amount_exp', [ + (8.0, 0.0, 10, 8), + (8.0, 0.0, 0, 8), + (8.0, 0.1, 0, 7.9), + (8.0, 0.1, 10, 8), + (8.0, 0.1, 8.0, 8.0), + (8.0, 0.1, 7.9, 7.9), +]) +def test_apply_fee_conditional(default_conf, fee, caplog, mocker, + amount, fee_abs, wallet, amount_exp): + walletmock = mocker.patch('freqtrade.wallets.Wallets.update') + # mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=wallet) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_order_id="123456" + ) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + walletmock.reset_mock() + # Amount is kept as is + assert freqtrade.apply_fee_conditional(trade, 'LTC', amount, fee_abs) == amount_exp + assert walletmock.call_count == 1 + + def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker, order_book_l2): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True From 81397874eb3a2002136f8ffdd544408fe8755c2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 May 2020 11:29:51 +0200 Subject: [PATCH 28/33] Remove commented mock --- tests/test_freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2c97b9fe1..11d9abb31 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3584,7 +3584,6 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): def test_apply_fee_conditional(default_conf, fee, caplog, mocker, amount, fee_abs, wallet, amount_exp): walletmock = mocker.patch('freqtrade.wallets.Wallets.update') - # mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=wallet) trade = Trade( pair='LTC/ETH', From e92d3867cfc54a95d3625c3c35c57a7538ec0111 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 May 2020 15:25:54 +0200 Subject: [PATCH 29/33] Fix failing test --- tests/test_freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 11d9abb31..b426368c9 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2202,6 +2202,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap limit_buy_order_old_partial_canceled, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=0)) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', From 0bd2fca40b648d6bef54388616832fce82528a59 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 10 May 2020 09:55:41 +0200 Subject: [PATCH 30/33] Update tests/test_integration.py Co-authored-by: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index ee6ef3cb2..90cdde61f 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -97,7 +97,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, # Only order for 3rd trade needs to be cancelled assert cancel_order_mock.call_count == 1 # Wallets must be updated between stoploss cancellation and selling, and will be updated again - # During update_trade_state + # during update_trade_state assert wallets_mock.call_count == 3 trade = trades[0] From 0c3bdd66aca0dab2ce912a547c57e24239babc95 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 13 May 2020 06:50:52 +0200 Subject: [PATCH 31/33] Update sql cheatsheet iwth current table structure --- docs/sql_cheatsheet.md | 79 +++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index b7b38c3dc..895a0536a 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -1,13 +1,20 @@ # SQL Helper + This page contains some help if you want to edit your sqlite db. ## Install sqlite3 -**Ubuntu/Debian installation** + +Sqlite3 is a terminal based sqlite application. +Feel free to use a visual Database editor like SqliteBrowser if you feel more comfortable with that. + +### Ubuntu/Debian installation + ```bash sudo apt-get install sqlite3 ``` ## Open the DB + ```bash sqlite3 .open @@ -16,45 +23,61 @@ sqlite3 ## Table structure ### List tables + ```bash .tables ``` ### Display table structure + ```bash .schema ``` ### Trade table structure + ```sql -CREATE TABLE trades ( - id INTEGER NOT NULL, - exchange VARCHAR NOT NULL, - pair VARCHAR NOT NULL, - is_open BOOLEAN NOT NULL, - fee_open FLOAT NOT NULL, - fee_close FLOAT NOT NULL, - open_rate FLOAT, - open_rate_requested FLOAT, - close_rate FLOAT, - close_rate_requested FLOAT, - close_profit FLOAT, - stake_amount FLOAT NOT NULL, - amount FLOAT, - open_date DATETIME NOT NULL, - close_date DATETIME, - open_order_id VARCHAR, - stop_loss FLOAT, - initial_stop_loss FLOAT, - stoploss_order_id VARCHAR, - stoploss_last_update DATETIME, - max_rate FLOAT, - sell_reason VARCHAR, - strategy VARCHAR, - ticker_interval INTEGER, - PRIMARY KEY (id), - CHECK (is_open IN (0, 1)) +CREATE TABLE trades + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee_open FLOAT NOT NULL, + fee_open_cost FLOAT, + fee_open_currency VARCHAR, + fee_close FLOAT NOT NULL, + fee_close_cost FLOAT, + fee_close_currency VARCHAR, + open_rate FLOAT, + open_rate_requested FLOAT, + open_trade_price FLOAT, + close_rate FLOAT, + close_rate_requested FLOAT, + close_profit FLOAT, + close_profit_abs FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + stop_loss FLOAT, + stop_loss_pct FLOAT, + initial_stop_loss FLOAT, + initial_stop_loss_pct FLOAT, + stoploss_order_id VARCHAR, + stoploss_last_update DATETIME, + max_rate FLOAT, + min_rate FLOAT, + sell_reason VARCHAR, + strategy VARCHAR, + ticker_interval INTEGER, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) ); +CREATE INDEX ix_trades_stoploss_order_id ON trades (stoploss_order_id); +CREATE INDEX ix_trades_pair ON trades (pair); +CREATE INDEX ix_trades_is_open ON trades (is_open); + ``` ## Get all trades in the table From 60f26ba5014bccb42f6df91fd776b1f4ab5745f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 13 May 2020 20:25:32 +0200 Subject: [PATCH 32/33] use update_trade_state also for closed stoploss orders --- freqtrade/freqtradebot.py | 59 ++++++++++++++++++++------------------ tests/test_freqtradebot.py | 3 +- tests/test_integration.py | 4 ++- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1f9573122..8732533f7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -752,7 +752,7 @@ class FreqtradeBot: # We check if stoploss order is fulfilled if stoploss_order and stoploss_order['status'] == 'closed': trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value - trade.update(stoploss_order) + self.update_trade_state(trade, stoploss_order, sl_order=True) # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) @@ -1123,7 +1123,7 @@ class FreqtradeBot: # def update_trade_state(self, trade: Trade, action_order: dict = None, - order_amount: float = None) -> bool: + order_amount: float = None, sl_order: bool = False) -> bool: """ Checks trades with open orders and updates the amount if necessary Handles closing both buy and sell orders. @@ -1131,34 +1131,37 @@ class FreqtradeBot: """ # Get order details for actual price per unit if trade.open_order_id: - # Update trade with order values - logger.info('Found open order for %s', trade) - try: - order = action_order or self.exchange.get_order(trade.open_order_id, trade.pair) - except InvalidOrderException as exception: - logger.warning('Unable to fetch order %s: %s', trade.open_order_id, exception) - return False - # Try update amount (binance-fix) - try: - new_amount = self.get_real_amount(trade, order, order_amount) - if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): - order['amount'] = new_amount - order.pop('filled', None) - trade.recalc_open_trade_price() - except DependencyException as exception: - logger.warning("Could not update trade amount: %s", exception) + order_id = trade.open_order_id + elif trade.stoploss_order_id and sl_order: + order_id = trade.stoploss_order_id + else: + return False + # Update trade with order values + logger.info('Found open order for %s', trade) + try: + order = action_order or self.exchange.get_order(order_id, trade.pair) + except InvalidOrderException as exception: + logger.warning('Unable to fetch order %s: %s', order_id, exception) + return False + # Try update amount (binance-fix) + try: + new_amount = self.get_real_amount(trade, order, order_amount) + if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): + order['amount'] = new_amount + order.pop('filled', None) + trade.recalc_open_trade_price() + except DependencyException as exception: + logger.warning("Could not update trade amount: %s", exception) - if self.exchange.check_order_canceled_empty(order): - # Trade has been cancelled on exchange - # Handling of this will happen in check_handle_timeout. - return True - trade.update(order) + if self.exchange.check_order_canceled_empty(order): + # Trade has been cancelled on exchange + # Handling of this will happen in check_handle_timeout. + return True + trade.update(order) - # Updating wallets when order is closed - if not trade.is_open: - self.wallets.update() - - return False + # Updating wallets when order is closed + if not trade.is_open: + self.wallets.update() def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, amount: float, fee_abs: float) -> float: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index b426368c9..2ff51dffd 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1140,7 +1140,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, 'status': 'closed', 'type': 'stop_loss_limit', 'price': 3, - 'average': 2 + 'average': 2, + 'amount': limit_buy_order['amount'], }) mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True diff --git a/tests/test_integration.py b/tests/test_integration.py index 90cdde61f..1396e86f5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -44,6 +44,8 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, } stoploss_order_closed = stoploss_order_open.copy() stoploss_order_closed['status'] = 'closed' + stoploss_order_closed['filled'] = stoploss_order_closed['amount'] + # Sell first trade based on stoploss, keep 2nd and 3rd trade open stoploss_order_mock = MagicMock( side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open]) @@ -98,7 +100,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, assert cancel_order_mock.call_count == 1 # Wallets must be updated between stoploss cancellation and selling, and will be updated again # during update_trade_state - assert wallets_mock.call_count == 3 + assert wallets_mock.call_count == 4 trade = trades[0] assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value From 255ff6cd0641d2f5796038d34ae4ead799eecff3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 13 May 2020 20:52:40 +0200 Subject: [PATCH 33/33] Should return False if it's not been cancelled empty --- freqtrade/freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8732533f7..7b012d7d3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1162,6 +1162,7 @@ class FreqtradeBot: # Updating wallets when order is closed if not trade.is_open: self.wallets.update() + return False def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, amount: float, fee_abs: float) -> float: