From c13eed2178819dfde379a803f13afca81869831d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Feb 2022 19:15:55 +0100 Subject: [PATCH 01/13] use Order object to update trade --- freqtrade/freqtradebot.py | 9 ++++--- freqtrade/persistence/models.py | 47 ++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5f2b72e1e..9166f43e9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,7 +16,7 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.enums import RPCMessageType, RunMode, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, - InvalidOrderException, PricingError) + InvalidOrderException, OperationalException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.mixins import LoggingMixin @@ -1358,8 +1358,11 @@ class FreqtradeBot(LoggingMixin): return True order = self.handle_order_fee(trade, order) - - trade.update(order) + order_obj = trade.select_order_by_order_id(order['id']) + if not order_obj: + # TODO: this can't happen! + raise OperationalException("order-obj not found!") + trade.update(order_obj) trade.recalc_trade_from_orders() Trade.commit() diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index b8ea5848f..e6daa08ba 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -113,14 +113,15 @@ class Order(_DECL_BASE): trade = relationship("Trade", back_populates="orders") - ft_order_side = Column(String(25), nullable=False) - ft_pair = Column(String(25), nullable=False) + # order_side can only be 'buy', 'sell' or 'stoploss' + ft_order_side: str = Column(String(25), nullable=False) + ft_pair: str = Column(String(25), nullable=False) ft_is_open = Column(Boolean, nullable=False, default=True, index=True) order_id = Column(String(255), nullable=False, index=True) status = Column(String(255), nullable=True) symbol = Column(String(25), nullable=True) - order_type = Column(String(50), nullable=True) + order_type: str = Column(String(50), nullable=True) side = Column(String(25), nullable=True) price = Column(Float, nullable=True) average = Column(Float, nullable=True) @@ -133,9 +134,18 @@ class Order(_DECL_BASE): order_update_date = Column(DateTime, nullable=True) @property - def order_date_utc(self): + def order_date_utc(self) -> datetime: + """ Order-date with UTC timezoneinfo""" return self.order_date.replace(tzinfo=timezone.utc) + @property + def safe_price(self) -> float: + return self.average or self.price + + @property + def safe_filled(self) -> float: + return self.filled or self.amount + def __repr__(self): return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' @@ -452,40 +462,39 @@ class LocalTrade(): f"Trailing stoploss saved us: " f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.") - def update(self, order: Dict) -> None: + def update(self, order: Order) -> None: """ Updates this entity with amount and actual open/close rates. :param order: order retrieved by exchange.fetch_order() :return: None """ - order_type = order['type'] # Ignore open and cancelled orders - if order['status'] == 'open' or safe_value_fallback(order, 'average', 'price') is None: + if order.status == 'open' or safe_value_fallback(order, 'average', 'price') is None: return - logger.info('Updating trade (id=%s) ...', self.id) + logger.info(f'Updating trade (id={self.id}) ...') - if order_type in ('market', 'limit') and order['side'] == 'buy': + if order.ft_order_side == 'buy': # Update open rate and actual amount - self.open_rate = float(safe_value_fallback(order, 'average', 'price')) - self.amount = float(safe_value_fallback(order, 'filled', 'amount')) + self.open_rate = order.safe_price + self.amount = order.safe_filled if self.is_open: - logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.') + logger.info(f'{order.order_type.upper()}_BUY has been fulfilled for {self}.') self.open_order_id = None self.recalc_trade_from_orders() - elif order_type in ('market', 'limit') and order['side'] == 'sell': + elif order.ft_order_side == 'sell': if self.is_open: - logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.') - self.close(safe_value_fallback(order, 'average', 'price')) - elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'): + logger.info(f'{order.order_type.upper()}_SELL has been fulfilled for {self}.') + self.close(order.safe_price) + elif order.ft_order_side == 'stoploss': self.stoploss_order_id = None self.close_rate_requested = self.stop_loss self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value if self.is_open: - logger.info(f'{order_type.upper()} is hit for {self}.') - self.close(safe_value_fallback(order, 'average', 'price')) + logger.info(f'{order.order_type.upper()} is hit for {self}.') + self.close(order.safe_price) else: - raise ValueError(f'Unknown order type: {order_type}') + raise ValueError(f'Unknown order type: {order.order_type}') Trade.commit() def close(self, rate: float, *, show_msg: bool = True) -> None: From 1b1216fc87b49050ab81e64f41ee346cbcaee04b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Feb 2022 19:18:19 +0100 Subject: [PATCH 02/13] Rename update_trade method --- freqtrade/freqtradebot.py | 3 ++- freqtrade/persistence/models.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9166f43e9..b8dfc6b08 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1362,7 +1362,8 @@ class FreqtradeBot(LoggingMixin): if not order_obj: # TODO: this can't happen! raise OperationalException("order-obj not found!") - trade.update(order_obj) + trade.update_trade(order_obj) + # TODO: is the below necessary? it's already done in update_trade for filled buys trade.recalc_trade_from_orders() Trade.commit() diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index e6daa08ba..52aea387f 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -151,7 +151,7 @@ class Order(_DECL_BASE): return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' f'side={self.side}, order_type={self.order_type}, status={self.status})') - def update_from_ccxt_object(self, order): + def update_from_ccxt_object(self, order) -> 'Order': """ Update Order from ccxt response Only updates if fields are available from ccxt - @@ -178,6 +178,7 @@ class Order(_DECL_BASE): if (order.get('filled', 0.0) or 0.0) > 0: self.order_filled_date = datetime.now(timezone.utc) self.order_update_date = datetime.now(timezone.utc) + return self def to_json(self) -> Dict[str, Any]: return { @@ -462,14 +463,14 @@ class LocalTrade(): f"Trailing stoploss saved us: " f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.") - def update(self, order: Order) -> None: + def update_trade(self, order: Order) -> None: """ Updates this entity with amount and actual open/close rates. :param order: order retrieved by exchange.fetch_order() :return: None """ # Ignore open and cancelled orders - if order.status == 'open' or safe_value_fallback(order, 'average', 'price') is None: + if order.status == 'open' or order.safe_price is None: return logger.info(f'Updating trade (id={self.id}) ...') From 508e677d70d8f4fd5825d94055c32280fecb9ffd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Feb 2022 20:40:50 +0100 Subject: [PATCH 03/13] Fix some tests to call update_trade with order object --- freqtrade/persistence/models.py | 1 - tests/conftest.py | 2 +- tests/test_freqtradebot.py | 39 +++++++++++++++++-------- tests/test_persistence.py | 50 ++++++++++++++++++++++----------- 4 files changed, 62 insertions(+), 30 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 52aea387f..731d57262 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -16,7 +16,6 @@ from sqlalchemy.sql.schema import UniqueConstraint from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES from freqtrade.enums import SellType from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.misc import safe_value_fallback from freqtrade.persistence.migrations import check_migrate diff --git a/tests/conftest.py b/tests/conftest.py index 630223d55..043cc6fcf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2208,7 +2208,7 @@ def limit_sell_order_usdt_open(): 'id': 'mocked_limit_sell_usdt', 'type': 'limit', 'side': 'sell', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'timestamp': arrow.utcnow().int_timestamp, 'price': 2.20, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 08d98b42d..50d060a39 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -227,7 +227,8 @@ def test_edge_overrides_stoploss(limit_buy_order_usdt, fee, caplog, mocker, freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) ############################################# # stoploss shoud be hit @@ -292,7 +293,8 @@ def test_create_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, assert trade.exchange == 'binance' # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) assert trade.open_rate == 2.0 assert trade.amount == 30.0 @@ -1803,7 +1805,8 @@ def test_handle_trade(default_conf_usdt, limit_buy_order_usdt, limit_sell_order_ assert trade time.sleep(0.01) # Race condition fix - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) assert trade.is_open is True freqtrade.wallets.update() @@ -1812,7 +1815,9 @@ def test_handle_trade(default_conf_usdt, limit_buy_order_usdt, limit_sell_order_ assert trade.open_order_id == limit_sell_order_usdt['id'] # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order_usdt) + oobj = Order.parse_from_ccxt_object( + limit_sell_order_usdt, limit_sell_order_usdt['symbol'], 'sell') + trade.update_trade(oobj) assert trade.close_rate == 2.2 assert trade.close_profit == 0.09451372 @@ -1962,8 +1967,11 @@ def test_close_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, trade = Trade.query.first() assert trade - trade.update(limit_buy_order_usdt) - trade.update(limit_sell_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) + oobj = Order.parse_from_ccxt_object( + limit_sell_order_usdt, limit_sell_order_usdt['symbol'], 'sell') + trade.update_trade(oobj) assert trade.is_open is False with pytest.raises(DependencyException, match=r'.*closed trade.*'): @@ -3103,7 +3111,8 @@ def test_sell_profit_only( freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True, None, None)) assert freqtrade.handle_trade(trade) is handle_first @@ -3139,7 +3148,9 @@ def test_sell_not_enough_balance(default_conf_usdt, limit_buy_order_usdt, limit_ trade = Trade.query.first() amnt = trade.amount - trade.update(limit_buy_order_usdt) + + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) patch_get_signal(freqtrade, value=(False, True, None, None)) mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=trade.amount * 0.985)) @@ -3247,7 +3258,8 @@ def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) freqtrade.wallets.update() patch_get_signal(freqtrade, value=(True, True, None, None)) assert freqtrade.handle_trade(trade) is False @@ -3437,7 +3449,8 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usd freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) # Sell due to min_roi_reached patch_get_signal(freqtrade, value=(True, False, None, None)) assert freqtrade.handle_trade(trade) is True @@ -3812,7 +3825,8 @@ def test_order_book_depth_of_market( assert len(Trade.query.all()) == 1 # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) assert trade.open_rate == 2.0 assert whitelist == default_conf_usdt['exchange']['pair_whitelist'] @@ -3906,7 +3920,8 @@ def test_order_book_ask_strategy( assert trade time.sleep(0.01) # Race condition fix - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) freqtrade.wallets.update() assert trade.is_open is True diff --git a/tests/test_persistence.py b/tests/test_persistence.py index b8f7a3336..3fa47daa9 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -108,7 +108,8 @@ def test_update_limit_order(limit_buy_order_usdt, limit_sell_order_usdt, fee, ca assert trade.close_date is None trade.open_order_id = 'something' - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) assert trade.open_order_id is None assert trade.open_rate == 2.00 assert trade.close_profit is None @@ -119,7 +120,8 @@ def test_update_limit_order(limit_buy_order_usdt, limit_sell_order_usdt, fee, ca caplog.clear() trade.open_order_id = 'something' - trade.update(limit_sell_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, 'ADA/USDT', 'sell') + trade.update_trade(oobj) assert trade.open_order_id is None assert trade.close_rate == 2.20 assert trade.close_profit == round(0.0945137157107232, 8) @@ -146,7 +148,8 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, ) trade.open_order_id = 'something' - trade.update(market_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(market_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) assert trade.open_order_id is None assert trade.open_rate == 2.0 assert trade.close_profit is None @@ -158,7 +161,8 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, caplog.clear() trade.is_open = True trade.open_order_id = 'something' - trade.update(market_sell_order_usdt) + oobj = Order.parse_from_ccxt_object(market_sell_order_usdt, 'ADA/USDT', 'sell') + trade.update_trade(oobj) assert trade.open_order_id is None assert trade.close_rate == 2.2 assert trade.close_profit == round(0.0945137157107232, 8) @@ -181,9 +185,11 @@ def test_calc_open_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt ) trade.open_order_id = 'something' - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) assert trade._calc_open_trade_value() == 60.15 - trade.update(limit_sell_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, 'ADA/USDT', 'sell') + trade.update_trade(oobj) assert isclose(trade.calc_close_trade_value(), 65.835) # Profit in USDT @@ -236,7 +242,8 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee): ) trade.open_order_id = 'something' - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) assert trade.calc_close_trade_value() == 0.0 @@ -257,7 +264,8 @@ def test_update_open_order(limit_buy_order_usdt): assert trade.close_date is None limit_buy_order_usdt['status'] = 'open' - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) assert trade.open_order_id is None assert trade.close_profit is None @@ -276,8 +284,9 @@ def test_update_invalid_order(limit_buy_order_usdt): exchange='binance', ) limit_buy_order_usdt['type'] = 'invalid' + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'meep') with pytest.raises(ValueError, match=r'Unknown order type'): - trade.update(limit_buy_order_usdt) + trade.update_trade(oobj) @pytest.mark.usefixtures("init_persistence") @@ -304,7 +313,8 @@ def test_calc_open_trade_value(limit_buy_order_usdt, fee): exchange='binance', ) trade.open_order_id = 'open_trade' - trade.update(limit_buy_order_usdt) # Buy @ 2.0 + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) # Buy @ 2.0 # Get the open rate price with the standard fee rate assert trade._calc_open_trade_value() == 60.15 @@ -325,14 +335,16 @@ def test_calc_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt, fee exchange='binance', ) trade.open_order_id = 'close_trade' - trade.update(limit_buy_order_usdt) # Buy @ 2.0 + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) # Buy @ 2.0 # Get the close rate price with a custom close rate and a regular fee rate assert trade.calc_close_trade_value(rate=2.5) == 74.8125 # Get the close rate price with a custom close rate and a custom fee rate assert trade.calc_close_trade_value(rate=2.5, fee=0.003) == 74.775 # Test when we apply a Sell order, and ask price with a custom fee rate - trade.update(limit_sell_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, 'ADA/USDT', 'sell') + trade.update_trade(oobj) assert trade.calc_close_trade_value(fee=0.005) == 65.67 @@ -409,7 +421,9 @@ def test_calc_profit(limit_buy_order_usdt, limit_sell_order_usdt, fee): exchange='binance', ) trade.open_order_id = 'something' - trade.update(limit_buy_order_usdt) # Buy @ 2.0 + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + + trade.update_trade(oobj) # Buy @ 2.0 # Custom closing rate and regular fee rate # Higher than open rate - 2.1 quote @@ -424,7 +438,8 @@ def test_calc_profit(limit_buy_order_usdt, limit_sell_order_usdt, fee): assert trade.calc_profit(rate=1.9, fee=0.003) == round(-3.320999999999998, 8) # Test when we apply a Sell order. Sell higher than open rate @ 2.2 - trade.update(limit_sell_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, 'ADA/USDT', 'sell') + trade.update_trade(oobj) assert trade.calc_profit() == round(5.684999999999995, 8) # Test with a custom fee rate on the close trade @@ -443,7 +458,9 @@ def test_calc_profit_ratio(limit_buy_order_usdt, limit_sell_order_usdt, fee): exchange='binance' ) trade.open_order_id = 'something' - trade.update(limit_buy_order_usdt) # Buy @ 2.0 + + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + trade.update_trade(oobj) # Buy @ 2.0 # Higher than open rate - 2.1 quote assert trade.calc_profit_ratio(rate=2.1) == round(0.04476309226932673, 8) @@ -457,7 +474,8 @@ def test_calc_profit_ratio(limit_buy_order_usdt, limit_sell_order_usdt, fee): assert trade.calc_profit_ratio(rate=1.9, fee=0.003) == round(-0.05521197007481293, 8) # Test when we apply a Sell order. Sell higher than open rate @ 2.2 - trade.update(limit_sell_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, 'ADA/USDT', 'sell') + trade.update_trade(oobj) assert trade.calc_profit_ratio() == round(0.0945137157107232, 8) # Test with a custom fee rate on the close trade From 874c161f78ccda10c76eac9c8d092e10d8c96ce8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 11 Feb 2022 07:33:46 +0100 Subject: [PATCH 04/13] Update more tests to use order_obj to update trade --- freqtrade/freqtradebot.py | 2 +- tests/conftest.py | 2 +- tests/rpc/test_rpc.py | 53 +++++++++++++++--------- tests/rpc/test_rpc_telegram.py | 74 +++++++++++++++++++++------------- tests/test_freqtradebot.py | 47 +++++++++++++++------ 5 files changed, 117 insertions(+), 61 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b8dfc6b08..06fcb8a9a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1361,7 +1361,7 @@ class FreqtradeBot(LoggingMixin): order_obj = trade.select_order_by_order_id(order['id']) if not order_obj: # TODO: this can't happen! - raise OperationalException("order-obj not found!") + raise OperationalException(f"order-obj for {order['id']} not found!") trade.update_trade(order_obj) # TODO: is the below necessary? it's already done in update_trade for filled buys trade.recalc_trade_from_orders() diff --git a/tests/conftest.py b/tests/conftest.py index 043cc6fcf..a7f23ea76 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1221,7 +1221,7 @@ def limit_sell_order_open(): 'id': 'mocked_limit_sell', 'type': 'limit', 'side': 'sell', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'timestamp': arrow.utcnow().int_timestamp, 'price': 0.00001173, diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 1c713ee86..e7b09ab74 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -11,6 +11,7 @@ from freqtrade.edge import PairInfo from freqtrade.enums import State from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError from freqtrade.persistence import Trade +from freqtrade.persistence.models import Order from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter @@ -277,8 +278,10 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, assert trade # Simulate buy & sell - trade.update(limit_buy_order) - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -415,28 +418,32 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, freqtradebot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'sell') + trade.update_trade(oobj) # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_sell_up ) - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False freqtradebot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_sell_up ) - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -495,14 +502,16 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, freqtradebot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_sell_up, get_fee=fee ) - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -754,13 +763,13 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: mocker.patch( 'freqtrade.exchange.Exchange.fetch_order', side_effect=[{ - 'id': '1234', + 'id': trade.orders[0].order_id, 'status': 'open', 'type': 'limit', 'side': 'buy', 'filled': filled_amount }, { - 'id': '1234', + 'id': trade.orders[0].order_id, 'status': 'closed', 'type': 'limit', 'side': 'buy', @@ -840,10 +849,12 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -874,10 +885,12 @@ def test_buy_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -946,10 +959,12 @@ def test_sell_reason_performance_handle(default_conf, ticker, limit_buy_order, f assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -1018,10 +1033,12 @@ def test_mix_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 640f9305c..44107db81 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -418,10 +418,12 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobjs) trade.close_date = datetime.utcnow() trade.is_open = False @@ -461,8 +463,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, trades = Trade.query.all() for trade in trades: - trade.update(limit_buy_order) - trade.update(limit_sell_order) + trade.update_trade(oobj) + trade.update_trade(oobjs) trade.close_date = datetime.utcnow() trade.is_open = False @@ -527,10 +529,12 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobjs) trade.close_date = datetime.utcnow() trade.is_open = False @@ -574,8 +578,8 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, trades = Trade.query.all() for trade in trades: - trade.update(limit_buy_order) - trade.update(limit_sell_order) + trade.update_trade(oobj) + trade.update_trade(oobjs) trade.close_date = datetime.utcnow() trade.is_open = False @@ -643,10 +647,12 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobjs) trade.close_date = datetime.utcnow() trade.is_open = False @@ -690,8 +696,8 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, trades = Trade.query.all() for trade in trades: - trade.update(limit_buy_order) - trade.update(limit_sell_order) + trade.update_trade(oobj) + trade.update_trade(oobjs) trade.close_date = datetime.utcnow() trade.is_open = False @@ -761,7 +767,9 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) + context = MagicMock() # Test with invalid 2nd argument (should silently pass) context.args = ["aaa"] @@ -776,7 +784,9 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up) - trade.update(limit_sell_order) + # Simulate fulfilled LIMIT_SELL order for trade + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.now(timezone.utc) trade.is_open = False @@ -1286,10 +1296,12 @@ def test_telegram_performance_handle(default_conf, update, ticker, fee, assert trade # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -1313,13 +1325,15 @@ def test_telegram_buy_tag_performance_handle(default_conf, update, ticker, fee, freqtradebot.enter_positions() trade = Trade.query.first() assert trade + trade.buy_tag = "TESTBUY" # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) - trade.buy_tag = "TESTBUY" # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -1356,13 +1370,14 @@ def test_telegram_sell_reason_performance_handle(default_conf, update, ticker, f freqtradebot.enter_positions() trade = Trade.query.first() assert trade - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - trade.sell_reason = 'TESTSELL' + # Simulate fulfilled LIMIT_BUY order for trade + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) + # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False @@ -1399,15 +1414,16 @@ def test_telegram_mix_tag_performance_handle(default_conf, update, ticker, fee, freqtradebot.enter_positions() trade = Trade.query.first() assert trade - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - trade.buy_tag = "TESTBUY" trade.sell_reason = "TESTSELL" + # Simulate fulfilled LIMIT_BUY order for trade + oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') + trade.update_trade(oobj) + # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + trade.update_trade(oobj) trade.close_date = datetime.utcnow() trade.is_open = False diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 50d060a39..79f297bff 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -984,11 +984,17 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, trade = Trade.query.first() trade.is_open = True trade.open_order_id = None - trade.stoploss_order_id = 100 + trade.stoploss_order_id = "100" + trade.orders.append(Order( + ft_order_side='stoploss', + order_id='100', + ft_pair=trade.pair, + ft_is_open=True, + )) assert trade stoploss_order_hit = MagicMock(return_value={ - 'id': 100, + 'id': "100", 'status': 'closed', 'type': 'stop_loss_limit', 'price': 3, @@ -1634,9 +1640,9 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_buy_order_usdt, cap mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=limit_buy_order_usdt['amount']) - + order_id = limit_buy_order_usdt['id'] trade = Trade( - open_order_id=123, + open_order_id=order_id, fee_open=0.001, fee_close=0.001, open_rate=0.01, @@ -1644,29 +1650,35 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_buy_order_usdt, cap amount=11, exchange="binance", ) + trade.orders.append(Order( + ft_order_side='buy', + price=0.01, + order_id=order_id, + + )) assert not freqtrade.update_trade_state(trade, None) assert log_has_re(r'Orderid for trade .* is empty.', caplog) caplog.clear() # Add datetime explicitly since sqlalchemy defaults apply only once written to database - freqtrade.update_trade_state(trade, '123') + freqtrade.update_trade_state(trade, order_id) # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) caplog.clear() assert trade.open_order_id is None assert trade.amount == limit_buy_order_usdt['amount'] - trade.open_order_id = '123' + trade.open_order_id = order_id mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) assert trade.amount != 90.81 # test amount modified by fee-logic - freqtrade.update_trade_state(trade, '123') + freqtrade.update_trade_state(trade, order_id) assert trade.amount == 90.81 assert trade.open_order_id is None trade.is_open = True trade.open_order_id = None # Assert we call handle_trade() if trade is feasible for execution - freqtrade.update_trade_state(trade, '123') + freqtrade.update_trade_state(trade, order_id) assert log_has_re('Found open order for.*', caplog) limit_buy_order_usdt_new = deepcopy(limit_buy_order_usdt) @@ -1675,7 +1687,7 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_buy_order_usdt, cap mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', side_effect=ValueError) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt_new) - res = freqtrade.update_trade_state(trade, '123') + res = freqtrade.update_trade_state(trade, order_id) # Cancelled empty assert res is True @@ -1687,6 +1699,8 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_buy_order_usdt, cap def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, limit_buy_order_usdt, fee, mocker, initial_amount, has_rounding_fee, caplog): trades_for_order[0]['amount'] = initial_amount + order_id = "oid_123456" + limit_buy_order_usdt['id'] = order_id mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) @@ -1702,10 +1716,18 @@ def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, l open_date=arrow.utcnow().datetime, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456", + open_order_id=order_id, is_open=True, ) - freqtrade.update_trade_state(trade, '123456', limit_buy_order_usdt) + trade.orders.append( + Order( + ft_order_side='buy', + ft_pair=trade.pair, + ft_is_open=True, + order_id=order_id, + ) + ) + freqtrade.update_trade_state(trade, order_id, limit_buy_order_usdt) assert trade.amount != amount assert trade.amount == limit_buy_order_usdt['amount'] if has_rounding_fee: @@ -3362,7 +3384,8 @@ def test_trailing_stop_loss_positive( freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order_usdt) + oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy') + trade.update_trade(oobj) caplog.set_level(logging.DEBUG) # stop-loss not reached assert freqtrade.handle_trade(trade) is False From e9f451406c4209ee7a8903af97aeda5b62b0b132 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 11 Feb 2022 07:43:45 +0100 Subject: [PATCH 05/13] Use correct order id --- freqtrade/freqtradebot.py | 4 ++-- tests/test_freqtradebot.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 06fcb8a9a..f2b076532 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1358,10 +1358,10 @@ class FreqtradeBot(LoggingMixin): return True order = self.handle_order_fee(trade, order) - order_obj = trade.select_order_by_order_id(order['id']) + order_obj = trade.select_order_by_order_id(order_id) if not order_obj: # TODO: this can't happen! - raise OperationalException(f"order-obj for {order['id']} not found!") + raise OperationalException(f"order-obj for {order_id} not found!") trade.update_trade(order_obj) # TODO: is the below necessary? it's already done in update_trade for filled buys trade.recalc_trade_from_orders() diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 79f297bff..3effce2f5 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1786,7 +1786,7 @@ def test_update_trade_state_sell(default_conf_usdt, trades_for_order, limit_sell fee_open=0.0025, fee_close=0.0025, open_date=arrow.utcnow().datetime, - open_order_id="123456", + open_order_id=limit_sell_order_usdt_open['id'], is_open=True, ) order = Order.parse_from_ccxt_object(limit_sell_order_usdt_open, 'LTC/ETH', 'sell') @@ -2016,7 +2016,7 @@ def test_bot_loop_start_called_once(mocker, default_conf_usdt, caplog): def test_check_handle_timedout_buy_usercustom(default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, fee, mocker) -> None: default_conf_usdt["unfilledtimeout"] = {"buy": 1400, "sell": 30} - + limit_buy_order_old['id'] = open_trade.open_order_id rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order_old) cancel_buy_order = deepcopy(limit_buy_order_old) From db540dc99078fce9ca48652f472ca43fba8cf0ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Feb 2022 14:21:22 +0100 Subject: [PATCH 06/13] Orders should also store fee if in receiving currency --- freqtrade/freqtradebot.py | 10 ++++++---- freqtrade/persistence/models.py | 16 +++++++++++++--- tests/test_freqtradebot.py | 10 ++++++++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f2b076532..e44193f21 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1357,8 +1357,8 @@ class FreqtradeBot(LoggingMixin): # Handling of this will happen in check_handle_timedout. return True - order = self.handle_order_fee(trade, order) order_obj = trade.select_order_by_order_id(order_id) + order = self.handle_order_fee(trade, order_obj, order) if not order_obj: # TODO: this can't happen! raise OperationalException(f"order-obj for {order_id} not found!") @@ -1415,14 +1415,16 @@ class FreqtradeBot(LoggingMixin): return real_amount return amount - def handle_order_fee(self, trade: Trade, order: Dict[str, Any]) -> Dict[str, Any]: + def handle_order_fee(self, trade: Trade, order_obj: Order, order: Dict[str, Any]) -> Dict[str, Any]: # Try update amount (binance-fix) try: new_amount = self.get_real_amount(trade, order) if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount, abs_tol=constants.MATH_CLOSE_PREC): - order['amount'] = new_amount - order.pop('filled', None) + # TODO: ?? + # order['amount'] = new_amount + order_obj.ft_fee_base = trade.amount - new_amount + # order.pop('filled', None) except DependencyException as exception: logger.warning("Could not update trade amount: %s", exception) return order diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 731d57262..0ee50d1c7 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -132,6 +132,8 @@ class Order(_DECL_BASE): order_filled_date = Column(DateTime, nullable=True) order_update_date = Column(DateTime, nullable=True) + ft_fee_base = Column(Float, nullable=True) + @property def order_date_utc(self) -> datetime: """ Order-date with UTC timezoneinfo""" @@ -143,7 +145,15 @@ class Order(_DECL_BASE): @property def safe_filled(self) -> float: - return self.filled or self.amount + return self.filled or self.amount or 0.0 + + @property + def safe_fee_base(self) -> float: + return self.ft_fee_base or 0.0 + + @property + def safe_amount_after_fee(self) -> float: + return self.safe_filled - self.safe_fee_base def __repr__(self): @@ -477,7 +487,7 @@ class LocalTrade(): if order.ft_order_side == 'buy': # Update open rate and actual amount self.open_rate = order.safe_price - self.amount = order.safe_filled + self.amount = order.safe_amount_after_fee if self.is_open: logger.info(f'{order.order_type.upper()}_BUY has been fulfilled for {self}.') self.open_order_id = None @@ -637,7 +647,7 @@ class LocalTrade(): (o.status not in NON_OPEN_EXCHANGE_STATES)): continue - tmp_amount = o.amount + tmp_amount = o.safe_amount_after_fee tmp_price = o.average or o.price if o.filled is not None: tmp_amount = o.filled diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3effce2f5..735a95231 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1729,9 +1729,14 @@ def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, l ) freqtrade.update_trade_state(trade, order_id, limit_buy_order_usdt) assert trade.amount != amount - assert trade.amount == limit_buy_order_usdt['amount'] + log_text = r'Applying fee on amount for .*' if has_rounding_fee: - assert log_has_re(r'Applying fee on amount for .*', caplog) + assert pytest.approx(trade.amount) == 29.992 + assert log_has_re(log_text, caplog) + else: + assert pytest.approx(trade.amount) == limit_buy_order_usdt['amount'] + assert not log_has_re(log_text, caplog) + def test_update_trade_state_exception(mocker, default_conf_usdt, @@ -2319,6 +2324,7 @@ def test_check_handle_timedout_partial_fee(default_conf_usdt, ticker_usdt, open_ limit_buy_order_old_partial_canceled, mocker) -> None: rpc_mock = patch_RPCManager(mocker) limit_buy_order_old_partial['id'] = open_trade.open_order_id + limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled) mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=0)) patch_exchange(mocker) From dc7bcf5dda0c94e6cab7e7943c8475522de9c462 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Feb 2022 14:27:26 +0100 Subject: [PATCH 07/13] Update failing test --- freqtrade/freqtradebot.py | 3 ++- freqtrade/persistence/models.py | 1 - tests/test_freqtradebot.py | 1 - tests/test_integration.py | 7 ++++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e44193f21..8007d520e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1415,7 +1415,8 @@ class FreqtradeBot(LoggingMixin): return real_amount return amount - def handle_order_fee(self, trade: Trade, order_obj: Order, order: Dict[str, Any]) -> Dict[str, Any]: + def handle_order_fee( + self, trade: Trade, order_obj: Order, order: Dict[str, Any]) -> Dict[str, Any]: # Try update amount (binance-fix) try: new_amount = self.get_real_amount(trade, order) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 0ee50d1c7..49ce34158 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -187,7 +187,6 @@ class Order(_DECL_BASE): if (order.get('filled', 0.0) or 0.0) > 0: self.order_filled_date = datetime.now(timezone.utc) self.order_update_date = datetime.now(timezone.utc) - return self def to_json(self) -> Dict[str, Any]: return { diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 735a95231..d7b47174b 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1738,7 +1738,6 @@ def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, l assert not log_has_re(log_text, caplog) - def test_update_trade_state_exception(mocker, default_conf_usdt, limit_buy_order_usdt, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) diff --git a/tests/test_integration.py b/tests/test_integration.py index ed38f1fec..db3b1b5fc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -4,6 +4,7 @@ import pytest from freqtrade.enums import SellType from freqtrade.persistence import Trade +from freqtrade.persistence.models import Order from freqtrade.rpc.rpc import RPC from freqtrade.strategy.interface import SellCheckTuple from tests.conftest import get_patched_freqtradebot, patch_get_signal @@ -94,7 +95,11 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, trades = Trade.query.all() # Make sure stoploss-order is open and trade is bought (since we mock update_trade_state) for trade in trades: - trade.stoploss_order_id = 3 + stoploss_order_closed['id'] = '3' + oobj = Order.parse_from_ccxt_object(stoploss_order_closed, trade.pair, 'stoploss') + + trade.orders.append(oobj) + trade.stoploss_order_id = '3' trade.open_order_id = None n = freqtrade.exit_positions(trades) From 6fb5b22a8e6acd41fb1a9eeb8fc2e6a4fd7d4bd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Feb 2022 15:36:25 +0100 Subject: [PATCH 08/13] Some cleanup --- freqtrade/freqtradebot.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8007d520e..e43f2a158 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,7 +16,7 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.enums import RPCMessageType, RunMode, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, - InvalidOrderException, OperationalException, PricingError) + InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.mixins import LoggingMixin @@ -1358,10 +1358,8 @@ class FreqtradeBot(LoggingMixin): return True order_obj = trade.select_order_by_order_id(order_id) - order = self.handle_order_fee(trade, order_obj, order) - if not order_obj: - # TODO: this can't happen! - raise OperationalException(f"order-obj for {order_id} not found!") + self.handle_order_fee(trade, order_obj, order) + trade.update_trade(order_obj) # TODO: is the below necessary? it's already done in update_trade for filled buys trade.recalc_trade_from_orders() @@ -1415,20 +1413,15 @@ class FreqtradeBot(LoggingMixin): return real_amount return amount - def handle_order_fee( - self, trade: Trade, order_obj: Order, order: Dict[str, Any]) -> Dict[str, Any]: + def handle_order_fee(self, trade: Trade, order_obj: Order, order: Dict[str, Any]) -> None: # Try update amount (binance-fix) try: new_amount = self.get_real_amount(trade, order) if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount, abs_tol=constants.MATH_CLOSE_PREC): - # TODO: ?? - # order['amount'] = new_amount order_obj.ft_fee_base = trade.amount - new_amount - # order.pop('filled', None) except DependencyException as exception: logger.warning("Could not update trade amount: %s", exception) - return order def get_real_amount(self, trade: Trade, order: Dict) -> float: """ From a24586cd41211b8dc99a9f70d3bc8d1c71a86d93 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Feb 2022 16:32:04 +0100 Subject: [PATCH 09/13] Update migrations for new column --- freqtrade/persistence/migrations.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 60c0eb5f9..288345e18 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -57,7 +57,7 @@ def set_sequence_ids(engine, order_id, trade_id): def migrate_trades_and_orders_table( decl_base, inspector, engine, trade_back_name: str, cols: List, - order_back_name: str): + order_back_name: str, cols_order: List): fee_open = get_column_def(cols, 'fee_open', 'fee') fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') @@ -141,7 +141,7 @@ def migrate_trades_and_orders_table( from {trade_back_name} """)) - migrate_orders_table(engine, order_back_name, cols) + migrate_orders_table(engine, order_back_name, cols_order) set_sequence_ids(engine, order_id, trade_id) @@ -171,17 +171,19 @@ def drop_orders_table(engine, table_back_name: str): connection.execute(text("drop table orders")) -def migrate_orders_table(engine, table_back_name: str, cols: List): +def migrate_orders_table(engine, table_back_name: str, cols_order: List): + + ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null') # let SQLAlchemy create the schema as required with engine.begin() as connection: connection.execute(text(f""" insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, average, remaining, cost, - order_date, order_filled_date, order_update_date) + order_date, order_filled_date, order_update_date, ft_fee_base) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, null average, remaining, cost, - order_date, order_filled_date, order_update_date + order_date, order_filled_date, order_update_date, {ft_fee_base} from {table_back_name} """)) @@ -193,6 +195,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') + cols_orders = inspector.get_columns('orders') tabs = get_table_names_for_table(inspector, 'trades') table_back_name = get_backup_name(tabs, 'trades_bak') order_tabs = get_table_names_for_table(inspector, 'orders') @@ -200,11 +203,12 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # Check if migration necessary # Migrates both trades and orders table! - if not has_column(cols, 'buy_tag'): + # if not has_column(cols, 'buy_tag'): + if 'orders' not in previous_tables or not has_column(cols_orders, 'ft_fee_base'): logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") migrate_trades_and_orders_table( - decl_base, inspector, engine, table_back_name, cols, order_table_bak_name) + decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_orders) # Reread columns - the above recreated the table! inspector = inspect(engine) cols = inspector.get_columns('trades') From fddacfedaae87fcf3c0b5e4ac5bba76d5cd7d50b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Feb 2022 16:34:35 +0100 Subject: [PATCH 10/13] Remove returntype --- freqtrade/freqtradebot.py | 3 +++ freqtrade/persistence/migrations.py | 3 --- freqtrade/persistence/models.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e43f2a158..99872ff0b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1358,6 +1358,9 @@ class FreqtradeBot(LoggingMixin): return True order_obj = trade.select_order_by_order_id(order_id) + if not order_obj: + raise DependencyException( + f"Order_obj not found for {order_id}. This should not have happened.") self.handle_order_fee(trade, order_obj, order) trade.update_trade(order_obj) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 288345e18..e4ff4bc37 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -209,9 +209,6 @@ def check_migrate(engine, decl_base, previous_tables) -> None: f"backup: {table_back_name}, {order_table_bak_name}") migrate_trades_and_orders_table( decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_orders) - # Reread columns - the above recreated the table! - inspector = inspect(engine) - cols = inspector.get_columns('trades') if 'orders' not in previous_tables and 'trades' in previous_tables: logger.info('Moving open orders to Orders table.') diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 49ce34158..d12a345e0 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -160,7 +160,7 @@ class Order(_DECL_BASE): return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' f'side={self.side}, order_type={self.order_type}, status={self.status})') - def update_from_ccxt_object(self, order) -> 'Order': + def update_from_ccxt_object(self, order): """ Update Order from ccxt response Only updates if fields are available from ccxt - From 7b6a0f7a19bb65ecb0d77a41fb808ed865f88404 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 05:28:52 +0000 Subject: [PATCH 11/13] Bump uvicorn from 0.17.4 to 0.17.5 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.17.4 to 0.17.5. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/uvicorn/compare/0.17.4...0.17.5) --- updated-dependencies: - dependency-name: uvicorn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5b4be4f37..7fa7db899 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ sdnotify==0.3.2 # API Server fastapi==0.74.0 -uvicorn==0.17.4 +uvicorn==0.17.5 pyjwt==2.3.0 aiofiles==0.8.0 psutil==5.9.0 From 731eb99713b8a7803b8496a2a6c44a9d92b7df32 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Feb 2022 19:17:50 +0100 Subject: [PATCH 12/13] Update mock-trade creation to rollback first --- tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 630223d55..8f1e75439 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -201,6 +201,9 @@ def create_mock_trades(fee, use_db: bool = True): """ Create some fake trades ... """ + if use_db: + Trade.query.session.rollback() + def add_trade(trade): if use_db: Trade.query.session.add(trade) From 42df65d4ec20196f1903e4ec4b7f9310f04ce39b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Feb 2022 06:25:21 +0100 Subject: [PATCH 13/13] Make sure backtesting is cleaned up in tests --- freqtrade/optimize/backtesting.py | 3 ++- tests/optimize/test_backtesting.py | 11 ++++++++--- tests/rpc/test_rpc_apiserver.py | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b423771ca..eca643732 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -128,7 +128,8 @@ class Backtesting: def __del__(self): self.cleanup() - def cleanup(self): + @staticmethod + def cleanup(): LoggingMixin.show_output = True PairLocks.use_db = True Trade.use_db = True diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index d61dffac4..a8998eb63 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -52,6 +52,13 @@ def trim_dictlist(dict_list, num): return new +@pytest.fixture(autouse=True) +def backtesting_cleanup() -> None: + yield None + + Backtesting.cleanup() + + def load_data_test(what, testdatadir): timerange = TimeRange.parse_timerange('1510694220-1510700340') data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir, @@ -553,8 +560,6 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: trade = backtesting._enter_trade(pair, row=row) assert trade is None - backtesting.cleanup() - def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: default_conf['use_sell_signal'] = False @@ -1423,7 +1428,7 @@ def test_get_strategy_run_id(default_conf_usdt): default_conf_usdt.update({ 'strategy': 'StrategyTestV2', 'max_open_trades': float('inf') - }) + }) strategy = StrategyResolver.load_strategy(default_conf_usdt) x = get_strategy_run_id(strategy) assert isinstance(x, str) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 5b19e5e05..544321860 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1108,6 +1108,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): data='{"tradeid": "1"}') assert_response(rc, 502) assert rc.json() == {"error": "Error querying /api/v1/forcesell: invalid argument"} + Trade.query.session.rollback() ftbot.enter_positions()