From 410324ac1926563f58f9de7fe229b9751ba2d90e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 10:13:21 +0000 Subject: [PATCH 01/11] time-jump detection should happen on the trimmed dataframe Fixes comment in #7615 --- freqtrade/data/history/idatahandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 7626198dc..6637663ff 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -308,7 +308,7 @@ class IDataHandler(ABC): timerange=timerange_startup, candle_type=candle_type ) - if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data, True): + if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): return pairdf else: enddate = pairdf.iloc[-1]['date'] @@ -316,7 +316,7 @@ class IDataHandler(ABC): if timerange_startup: self._validate_pairdata(pair, pairdf, timeframe, candle_type, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) - if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): + if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data, True): return pairdf # incomplete candles should only be dropped if we didn't trim the end beforehand. From 1dc3c58775daa2593b6dcb920858556971380b68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 11:04:56 +0000 Subject: [PATCH 02/11] Convert missing candle count to int closes #8082 --- freqtrade/data/dataprovider.py | 12 +++++++----- tests/data/test_dataprovider.py | 5 +++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 3ebfd809c..60a058952 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -9,7 +9,7 @@ from collections import deque from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple -from pandas import DataFrame, to_timedelta +from pandas import DataFrame, Timedelta, Timestamp, to_timedelta from freqtrade.configuration import TimeRange from freqtrade.constants import (FULL_DATAFRAME_THRESHOLD, Config, ListPairsWithTimeframes, @@ -206,9 +206,11 @@ class DataProvider: existing_df, _ = self.__producer_pairs_df[producer_name][pair_key] # CHECK FOR MISSING CANDLES - timeframe_delta = to_timedelta(timeframe) # Convert the timeframe to a timedelta for pandas - local_last = existing_df.iloc[-1]['date'] # We want the last date from our copy - incoming_first = dataframe.iloc[0]['date'] # We want the first date from the incoming + # Convert the timeframe to a timedelta for pandas + timeframe_delta: Timedelta = to_timedelta(timeframe) + local_last: Timestamp = existing_df.iloc[-1]['date'] # We want the last date from our copy + # We want the first date from the incoming + incoming_first: Timestamp = dataframe.iloc[0]['date'] # Remove existing candles that are newer than the incoming first candle existing_df1 = existing_df[existing_df['date'] < incoming_first] @@ -221,7 +223,7 @@ class DataProvider: # we missed some candles between our data and the incoming # so return False and candle_difference. if candle_difference > 1: - return (False, candle_difference) + return (False, int(candle_difference)) if existing_df1.empty: appended_df = dataframe else: diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index e0c79d52a..c6b1dcc5a 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -437,6 +437,7 @@ def test_dp__add_external_df(default_conf_usdt): # Add the same dataframe again - dataframe size shall not change. res = dp._add_external_df('ETH/USDT', df, last_analyzed, timeframe, CandleType.SPOT) assert res[0] is True + assert isinstance(res[1], int) assert res[1] == 0 df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT) assert len(df) == 24 @@ -446,6 +447,7 @@ def test_dp__add_external_df(default_conf_usdt): res = dp._add_external_df('ETH/USDT', df2, last_analyzed, timeframe, CandleType.SPOT) assert res[0] is True + assert isinstance(res[1], int) assert res[1] == 0 df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT) assert len(df) == 48 @@ -455,6 +457,7 @@ def test_dp__add_external_df(default_conf_usdt): res = dp._add_external_df('ETH/USDT', df3, last_analyzed, timeframe, CandleType.SPOT) assert res[0] is True + assert isinstance(res[1], int) assert res[1] == 0 df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT) # New length = 48 + 12 (since we have a 12 hour offset). @@ -478,6 +481,7 @@ def test_dp__add_external_df(default_conf_usdt): res = dp._add_external_df('ETH/USDT', df4, last_analyzed, timeframe, CandleType.SPOT) assert res[0] is False # 36 hours - from 2022-01-03 12:00:00+00:00 to 2022-01-05 00:00:00+00:00 + assert isinstance(res[1], int) assert res[1] == 36 df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT) # New length = 61 + 1 @@ -488,4 +492,5 @@ def test_dp__add_external_df(default_conf_usdt): res = dp._add_external_df('ETH/USDT', df4, last_analyzed, timeframe, CandleType.SPOT) assert res[0] is False # 36 hours - from 2022-01-03 12:00:00+00:00 to 2022-01-05 00:00:00+00:00 + assert isinstance(res[1], int) assert res[1] == 0 From 2c1457fb95a306bf93274611a7e8793e7ec2f551 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 11:06:23 +0000 Subject: [PATCH 03/11] Ensure limit is integer (on server) --- freqtrade/rpc/api_server/api_ws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_ws.py b/freqtrade/rpc/api_server/api_ws.py index 18714f15f..b253d66c2 100644 --- a/freqtrade/rpc/api_server/api_ws.py +++ b/freqtrade/rpc/api_server/api_ws.py @@ -90,7 +90,7 @@ async def _process_consumer_request( elif type == RPCRequestType.ANALYZED_DF: # Limit the amount of candles per dataframe to 'limit' or 1500 - limit = min(data.get('limit', 1500), 1500) if data else None + limit = int(min(data.get('limit', 1500), 1500)) if data else None pair = data.get('pair', None) if data else None # For every pair in the generator, send a separate message From 5073c780d8ecbaefb52fce732af701f630d8f565 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 11:22:04 +0000 Subject: [PATCH 04/11] .agg would like strings, not the sum function. --- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py | 2 +- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py index 9520123ee..88c97989a 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py @@ -44,7 +44,7 @@ class SharpeHyperOptLossDaily(IHyperOptLoss): sum_daily = ( results.resample(resample_freq, on='close_date').agg( - {"profit_ratio_after_slippage": sum}).reindex(t_index).fillna(0) + {"profit_ratio_after_slippage": 'sum'}).reindex(t_index).fillna(0) ) total_profit = sum_daily["profit_ratio_after_slippage"] - risk_free_rate diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py index fac96664d..f5fe4590e 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py @@ -46,7 +46,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): sum_daily = ( results.resample(resample_freq, on='close_date').agg( - {"profit_ratio_after_slippage": sum}).reindex(t_index).fillna(0) + {"profit_ratio_after_slippage": 'sum'}).reindex(t_index).fillna(0) ) total_profit = sum_daily["profit_ratio_after_slippage"] - minimum_acceptable_return From 42f07e6ec21d2e1a776ede1e3a6af25196a7dbff Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 19:45:27 +0100 Subject: [PATCH 05/11] Improve order_parse tests --- tests/exchange/test_ccxt_compat.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 2ca92799f..5bd00d296 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -49,8 +49,8 @@ EXCHANGES = { "orderListId": -1, "clientOrderId": "x-R4DD3S8297c73a11ccb9dc8f2811ba", "transactTime": 1674493798550, - "price": "15.00000000", - "origQty": "1.00000000", + "price": "15.50000000", + "origQty": "1.10000000", "executedQty": "0.00000000", "cummulativeQuoteQty": "0.00000000", "status": "NEW", @@ -74,8 +74,8 @@ EXCHANGES = { "orderListId": -1, "clientOrderId": "x-R4DD3S8297c73a11ccb9dc8f2811ba", "transactTime": 1674493798550, - "price": "15.00000000", - "origQty": "1.00000000", + "price": "15.50000000", + "origQty": "1.10000000", "executedQty": "0.00000000", "cummulativeQuoteQty": "0.00000000", "status": "NEW", @@ -106,12 +106,12 @@ EXCHANGES = { {'id': '63d6742d0adc5570001d2bbf7'}, # create order { 'id': '63d6742d0adc5570001d2bbf7', - 'symbol': 'NAKA-USDT', + 'symbol': 'SOL-USDT', 'opType': 'DEAL', 'type': 'limit', 'side': 'buy', - 'price': '30', - 'size': '0.1', + 'price': '15.5', + 'size': '1.1', 'funds': '0', 'dealFunds': '0.032626', 'dealSize': '0.1', @@ -311,7 +311,10 @@ class TestCCXTExchange(): assert isinstance(po['datetime'], str) assert isinstance(po['timestamp'], int) assert isinstance(po['price'], float) + assert po['price'] == 15.5 + assert po['symbol'] == 'SOL/USDT' assert isinstance(po['amount'], float) + assert po['amount'] == 1.1 assert isinstance(po['status'], str) else: pytest.skip(f"No sample order available for exchange {exchange_name}") From 50d3b7bdefb3cc2e126226b9ff603486510fd6e2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 20:00:05 +0100 Subject: [PATCH 06/11] Add bybit sample order --- tests/exchange/test_ccxt_compat.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 5bd00d296..bc85f7cf7 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -168,6 +168,23 @@ EXCHANGES = { 'futures': True, 'leverage_tiers_public': True, 'leverage_in_spot_market': True, + 'sample_order': [ + { + "orderId": "1274754916287346280", + "orderLinkId": "1666798627015730", + "symbol": "SOLUSDT", + "createTime": "1674493798550", + "orderPrice": "15.5", + "orderQty": "1.1", + "orderType": "LIMIT", + "side": "BUY", + "status": "NEW", + "timeInForce": "GTC", + "accountId": "5555555", + "execQty": "0", + "orderCategory": "0" + } + ] }, 'huobi': { 'pair': 'ETH/BTC', From 448505fbfbf960aa5462822c21b57c5efac1fbd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 20:38:18 +0100 Subject: [PATCH 07/11] Fix minor issue where amount could be empty in rest calls --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index daeccb216..a112771d3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -172,7 +172,7 @@ class Order(_DECL_BASE): def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]: resp = { - 'amount': self.amount, + 'amount': self.safe_amount, 'safe_price': self.safe_price, 'ft_order_side': self.ft_order_side, 'order_filled_timestamp': int(self.order_filled_date.replace( From 680136f57d5421042c1dc62aee0f5f3951e53196 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 20:42:41 +0100 Subject: [PATCH 08/11] Add workaround patch for kucoin create_order returning empty While the actual problem is caused by a ccxt change, the change itself makes sense. once ccxt starts returning the correct status (open) for create-orders, we can remove the fix. closes #8079 --- freqtrade/exchange/kucoin.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/freqtrade/exchange/kucoin.py b/freqtrade/exchange/kucoin.py index 6c7d7acfc..797d9fbd2 100644 --- a/freqtrade/exchange/kucoin.py +++ b/freqtrade/exchange/kucoin.py @@ -36,3 +36,34 @@ class Kucoin(Exchange): 'stop': 'loss' }) return params + + def create_order( + self, + *, + pair: str, + ordertype: str, + side: BuySell, + amount: float, + rate: float, + leverage: float, + reduceOnly: bool = False, + time_in_force: str = 'GTC', + ) -> Dict: + + res = super().create_order( + pair=pair, + ordertype=ordertype, + side=side, + amount=amount, + rate=rate, + leverage=leverage, + reduceOnly=reduceOnly, + time_in_force=time_in_force, + ) + # Kucoin returns only the order-id. + # ccxt returns status = 'closed' at the moment - which is information ccxt invented. + # Since we rely on status heavily, we must set it to 'open' here. + # ref: https://github.com/ccxt/ccxt/pull/16674, (https://github.com/ccxt/ccxt/pull/16553) + res['type'] = ordertype + res['status'] = 'open' + return res From 8a0fabed0eeefde57546d5dc9c4f67bdd858a1b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 20:55:11 +0100 Subject: [PATCH 09/11] Ensure we don't overwrite valid values by invalid exchange responses --- freqtrade/freqtradebot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a558f7bf4..dbe513ccf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -750,13 +750,15 @@ class FreqtradeBot(LoggingMixin): self.exchange.name, order['filled'], order['amount'], order['remaining'] ) - amount = safe_value_fallback(order, 'filled', 'amount') - enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') + amount = safe_value_fallback(order, 'filled', 'amount', amount) + enter_limit_filled_price = safe_value_fallback( + order, 'average', 'price', enter_limit_filled_price) # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': - amount = safe_value_fallback(order, 'filled', 'amount') - enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') + amount = safe_value_fallback(order, 'filled', 'amount', amount) + enter_limit_filled_price = safe_value_fallback( + order, 'average', 'price', enter_limit_requested) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') From 9bb376296d06ae8884bec1eb82c4d06ea2d480c7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jan 2023 20:59:55 +0100 Subject: [PATCH 10/11] Update parse_order test --- tests/exchange/test_ccxt_compat.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index bc85f7cf7..c4757288d 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -323,16 +323,19 @@ class TestCCXTExchange(): po = exch._api.parse_order(order) assert isinstance(po['id'], str) assert po['id'] is not None - if len(order.keys()) > 1: - assert po['timestamp'] == 1674493798550 - assert isinstance(po['datetime'], str) - assert isinstance(po['timestamp'], int) - assert isinstance(po['price'], float) - assert po['price'] == 15.5 - assert po['symbol'] == 'SOL/USDT' - assert isinstance(po['amount'], float) - assert po['amount'] == 1.1 - assert isinstance(po['status'], str) + if len(order.keys()) < 5: + # Kucoin case + assert po['status'] == 'closed' + continue + assert po['timestamp'] == 1674493798550 + assert isinstance(po['datetime'], str) + assert isinstance(po['timestamp'], int) + assert isinstance(po['price'], float) + assert po['price'] == 15.5 + assert po['symbol'] == 'SOL/USDT' + assert isinstance(po['amount'], float) + assert po['amount'] == 1.1 + assert isinstance(po['status'], str) else: pytest.skip(f"No sample order available for exchange {exchange_name}") From 72a98943b1322da4bebfb148e47fe8c069c9878c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Feb 2023 06:55:51 +0100 Subject: [PATCH 11/11] bybit: Add correct funding_fee_timeframe --- freqtrade/exchange/bybit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index 55bfbd232..d0598d8de 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -34,6 +34,7 @@ class Bybit(Exchange): "ohlcv_candle_limit": 200, "ohlcv_has_history": True, "mark_ohlcv_timeframe": "4h", + "funding_fee_timeframe": "8h", "stoploss_on_exchange": True, "stoploss_order_types": {"limit": "limit", "market": "market"}, }