From df50b1928d6d58b130d1d2b40dcaad334800bcc3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Sep 2022 06:51:30 +0200 Subject: [PATCH 1/6] Fix funding fee calculation for backtesting --- freqtrade/optimize/backtesting.py | 2 +- freqtrade/persistence/trade_model.py | 34 +++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 77bf3d8ad..8f6b6b332 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -686,7 +686,7 @@ class Backtesting: self.futures_data[trade.pair], amount=trade.amount, is_short=trade.is_short, - open_date=trade.open_date_utc, + open_date=trade.date_last_filled_utc, close_date=exit_candle_time, ) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 1f14f110e..9c9c7f381 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -65,6 +65,8 @@ class Order(_DECL_BASE): order_filled_date = Column(DateTime, nullable=True) order_update_date = Column(DateTime, nullable=True) + funding_fee = Column(Float, nullable=True) + ft_fee_base = Column(Float, nullable=True) @property @@ -72,6 +74,13 @@ class Order(_DECL_BASE): """ Order-date with UTC timezoneinfo""" return self.order_date.replace(tzinfo=timezone.utc) + @property + def order_filled_utc(self) -> Optional[datetime]: + """ last order-date with UTC timezoneinfo""" + return ( + self.order_filled_date.replace(tzinfo=timezone.utc) if self.order_filled_date else None + ) + @property def safe_price(self) -> float: return self.average or self.price @@ -179,6 +188,10 @@ class Order(_DECL_BASE): self.remaining = 0 self.status = 'closed' self.ft_is_open = False + # Assign funding fees to Order. + # Assumes backtesting will use date_last_filled_utc to calculate future funding fees. + self.funding_fee = trade.funding_fees + if (self.ft_order_side == trade.entry_side): trade.open_rate = self.price trade.recalc_trade_from_orders() @@ -346,6 +359,12 @@ class LocalTrade(): else: return self.amount + @property + def date_last_filled_utc(self): + """ Date of the last filled order""" + return max([self.open_date_utc, + max(o.order_filled_utc for o in self.orders if o.filled and not o.ft_is_open)]) + @property def open_date_utc(self): return self.open_date.replace(tzinfo=timezone.utc) @@ -843,10 +862,14 @@ class LocalTrade(): close_profit = 0.0 close_profit_abs = 0.0 profit = None - for o in self.orders: + # Reset funding fees + self.funding_fees = 0.0 + funding_fees = 0.0 + ordercount = len(self.orders) - 1 + for i, o in enumerate(self.orders): if o.ft_is_open or not o.filled: continue - + funding_fees += (o.funding_fee or 0.0) tmp_amount = FtPrecise(o.safe_amount_after_fee) tmp_price = FtPrecise(o.safe_price) @@ -861,7 +884,11 @@ class LocalTrade(): avg_price = current_stake / current_amount if is_exit: - # Process partial exits + # Process exits + if i == ordercount and is_closing: + # Apply funding fees only to the last order + self.funding_fees = funding_fees + exit_rate = o.safe_price exit_amount = o.safe_amount_after_fee profit = self.calc_profit(rate=exit_rate, amount=exit_amount, @@ -871,6 +898,7 @@ class LocalTrade(): exit_rate, amount=exit_amount, open_rate=avg_price) else: total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price) + self.funding_fees = funding_fees if close_profit: self.close_profit = close_profit From 0c6a02687a49b48c433ec841200911b21efc8f24 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Sep 2022 20:55:13 +0200 Subject: [PATCH 2/6] Don't calculate funding fees if we're not going to use them. --- freqtrade/freqtradebot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 37bc6dfed..95b42155f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -726,10 +726,11 @@ class FreqtradeBot(LoggingMixin): fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') base_currency = self.exchange.get_pair_base_currency(pair) open_date = datetime.now(timezone.utc) - funding_fees = self.exchange.get_funding_fees( - pair=pair, amount=amount, is_short=is_short, open_date=open_date) + # This is a new trade if trade is None: + funding_fees = self.exchange.get_funding_fees( + pair=pair, amount=amount, is_short=is_short, open_date=open_date) trade = Trade( pair=pair, base_currency=base_currency, From 0f483ee31f043ceafaa7a5c19eaa77ade2e1d323 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Sep 2022 10:06:23 +0200 Subject: [PATCH 3/6] Use "since last order" approach for live as well. --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/persistence/trade_model.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 95b42155f..5323a5fc0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -281,7 +281,7 @@ class FreqtradeBot(LoggingMixin): pair=trade.pair, amount=trade.amount, is_short=trade.is_short, - open_date=trade.open_date_utc + open_date=trade.date_last_filled_utc ) trade.funding_fees = funding_fees else: @@ -1485,7 +1485,7 @@ class FreqtradeBot(LoggingMixin): pair=trade.pair, amount=trade.amount, is_short=trade.is_short, - open_date=trade.open_date_utc, + open_date=trade.date_last_filled_utc, ) exit_type = 'exit' exit_reason = exit_tag or exit_check.exit_reason diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 9c9c7f381..c2a87daaa 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -128,6 +128,10 @@ class Order(_DECL_BASE): self.ft_is_open = True if self.status in NON_OPEN_EXCHANGE_STATES: self.ft_is_open = False + if self.trade: + # Assign funding fee up to this point + # (represents the funding fee since the last order) + self.funding_fee = self.trade.funding_fees if (order.get('filled', 0.0) or 0.0) > 0: self.order_filled_date = datetime.now(timezone.utc) self.order_update_date = datetime.now(timezone.utc) @@ -360,10 +364,12 @@ class LocalTrade(): return self.amount @property - def date_last_filled_utc(self): + def date_last_filled_utc(self) -> datetime: """ Date of the last filled order""" - return max([self.open_date_utc, - max(o.order_filled_utc for o in self.orders if o.filled and not o.ft_is_open)]) + orders = self.select_filled_orders() + if not orders: + return self.open_date_utc + return max([self.open_date_utc, max(o.order_filled_utc for o in orders)]) @property def open_date_utc(self): From b95b3d8391f4df3bb3510173711349d7356a08ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Sep 2022 15:09:50 +0200 Subject: [PATCH 4/6] Update test to actually test funding fee appliance --- tests/test_persistence.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 23ccc67f3..cdca3bc4d 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -615,21 +615,25 @@ def test_calc_open_close_trade_price( is_short=is_short, leverage=lev, trading_mode=trading_mode, - funding_fees=funding_fees ) entry_order = limit_order[trade.entry_side] exit_order = limit_order[trade.exit_side] trade.open_order_id = f'something-{is_short}-{lev}-{exchange}' oobj = Order.parse_from_ccxt_object(entry_order, 'ADA/USDT', trade.entry_side) - trade.orders.append(oobj) + oobj.trade = trade + oobj.update_from_ccxt_object(entry_order) trade.update_trade(oobj) + trade.funding_fees = funding_fees + oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', trade.exit_side) - trade.orders.append(oobj) + oobj.trade = trade + oobj.update_from_ccxt_object(exit_order) trade.update_trade(oobj) assert trade.is_open is False + assert trade.funding_fees == funding_fees assert pytest.approx(trade._calc_open_trade_value(trade.amount, trade.open_rate)) == open_value assert pytest.approx(trade.calc_close_trade_value(trade.close_rate)) == close_value From ed4cc18cddaaff7bdb95738ee4a3d5093530b0e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Sep 2022 15:18:09 +0200 Subject: [PATCH 5/6] Migration to check order funding fee --- freqtrade/persistence/migrations.py | 10 ++++++---- freqtrade/persistence/trade_model.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 1131c88b4..1f1330ba7 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -212,17 +212,18 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null') average = get_column_def(cols_order, 'average', 'null') stop_price = get_column_def(cols_order, 'stop_price', 'null') + funding_fee = get_column_def(cols_order, 'funding_fee', '0.0') # sqlite does not support literals for booleans with engine.begin() as connection: connection.execute(text(f""" insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, average, remaining, cost, - stop_price, order_date, order_filled_date, order_update_date, ft_fee_base) + stop_price, order_date, order_filled_date, order_update_date, ft_fee_base, funding_fee) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, {average} average, remaining, cost, {stop_price} stop_price, order_date, order_filled_date, - order_update_date, {ft_fee_base} ft_fee_base + order_update_date, {ft_fee_base} ft_fee_base {funding_fee} funding_fee from {table_back_name} """)) @@ -307,9 +308,10 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # Check if migration necessary # Migrates both trades and orders table! # if ('orders' not in previous_tables - # or not has_column(cols_orders, 'stop_price')): + # or not has_column(cols_orders, 'funding_fee')): migrating = False - if not has_column(cols_trades, 'contract_size'): + if not has_column(cols_orders, 'funding_fee'): + # if not has_column(cols_trades, 'contract_size'): migrating = True logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index c2a87daaa..6127c502d 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -892,7 +892,7 @@ class LocalTrade(): if is_exit: # Process exits if i == ordercount and is_closing: - # Apply funding fees only to the last order + # Apply funding fees only to the last closing order self.funding_fees = funding_fees exit_rate = o.safe_price From 16573b19e3d478b3a5625a81d2504823dee4cdf8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Sep 2022 19:34:38 +0200 Subject: [PATCH 6/6] Fix migration syntax error --- freqtrade/persistence/migrations.py | 4 ++-- freqtrade/persistence/trade_model.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 1f1330ba7..a54c5570f 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -223,7 +223,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, {average} average, remaining, cost, {stop_price} stop_price, order_date, order_filled_date, - order_update_date, {ft_fee_base} ft_fee_base {funding_fee} funding_fee + order_update_date, {ft_fee_base} ft_fee_base, {funding_fee} funding_fee from {table_back_name} """)) @@ -310,8 +310,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # if ('orders' not in previous_tables # or not has_column(cols_orders, 'funding_fee')): migrating = False - if not has_column(cols_orders, 'funding_fee'): # if not has_column(cols_trades, 'contract_size'): + if not has_column(cols_orders, 'funding_fee'): migrating = True logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 6127c502d..ea60796a4 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -369,7 +369,8 @@ class LocalTrade(): orders = self.select_filled_orders() if not orders: return self.open_date_utc - return max([self.open_date_utc, max(o.order_filled_utc for o in orders)]) + return max([self.open_date_utc, + max(o.order_filled_utc for o in orders if o.order_filled_utc)]) @property def open_date_utc(self):