From 3f266e8c8cfafbcef39f87b06131294604f3e3eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Dec 2021 06:46:35 +0100 Subject: [PATCH 1/8] Improve ccxt_mark_price_test --- tests/exchange/test_ccxt_compat.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 8710463a6..0dda4ce52 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -213,24 +213,29 @@ class TestCCXTExchange(): assert rate[int(this_hour.timestamp() * 1000)] != 0.0 assert rate[int(prev_tick.timestamp() * 1000)] != 0.0 - @pytest.mark.skip("No futures support yet") - def test_fetch_mark_price_history(self, exchange_futures): + def test_ccxt_fetch_mark_price_history(self, exchange_futures): exchange, exchangename = exchange_futures if not exchange: # exchange_futures only returns values for supported exchanges return pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) since = int((datetime.now(timezone.utc) - timedelta(days=5)).timestamp() * 1000) + pair_tf = (pair, '1h', CandleType.MARK) - mark_candles = exchange._get_mark_price_history(pair, since) + mark_ohlcv = exchange.refresh_latest_ohlcv( + [pair_tf], + since_ms=since, + drop_incomplete=False) - assert isinstance(mark_candles, dict) + assert isinstance(mark_ohlcv, dict) expected_tf = '1h' + mark_candles = mark_ohlcv[pair_tf] this_hour = timeframe_to_prev_date(expected_tf) - prev_tick = timeframe_to_prev_date(expected_tf, this_hour - timedelta(minutes=1)) - assert mark_candles[int(this_hour.timestamp() * 1000)] != 0.0 - assert mark_candles[int(prev_tick.timestamp() * 1000)] != 0.0 + prev_hour = timeframe_to_prev_date(expected_tf, this_hour - timedelta(minutes=1)) + + assert mark_candles[mark_candles['date'] == prev_hour].iloc[0]['open'] != 0.0 + assert mark_candles[mark_candles['date'] == this_hour].iloc[0]['open'] != 0.0 # TODO: tests fetch_trades (?) From 35f9549e989a95de1daafc66c437c10e3e645e44 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Dec 2021 07:14:41 +0100 Subject: [PATCH 2/8] Expose drop_incomplete from refresh_latest_ohlcv --- freqtrade/exchange/exchange.py | 10 +++++++--- tests/exchange/test_ccxt_compat.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6aa15f550..e09ed1c52 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1389,7 +1389,8 @@ class Exchange: return pair, timeframe, candle_type, data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, - since_ms: Optional[int] = None, cache: bool = True + since_ms: Optional[int] = None, cache: bool = True, + drop_incomplete: bool = None ) -> Dict[PairWithTimeframe, DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result @@ -1398,10 +1399,13 @@ class Exchange: :param pair_list: List of 2 element tuples containing pair, interval to refresh :param since_ms: time since when to download, in milliseconds :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists + :param drop_incomplete: Control candle dropping. + Specifying None defaults to _ohlcv_partial_candle :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) - + # TODO-lev: maybe depend this on candle type? + drop_incomplete = self._ohlcv_partial_candle if drop_incomplete is None else drop_incomplete input_coroutines = [] cached_pairs = [] # Gather coroutines to run @@ -1447,7 +1451,7 @@ class Exchange: # keeping parsed dataframe in cache ohlcv_df = ohlcv_to_dataframe( ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) + drop_incomplete=drop_incomplete) results_df[(pair, timeframe, c_type)] = ohlcv_df if cache: self._klines[(pair, timeframe, c_type)] = ohlcv_df diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 0dda4ce52..c9b2173b6 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -91,7 +91,7 @@ def exchange_futures(request, exchange_conf, class_mocker): exchange_conf['exchange']['name'] = request.param exchange_conf['trading_mode'] = 'futures' exchange_conf['collateral'] = 'cross' - # TODO-lev This mock should no longer be necessary once futures are enabled. + # TODO-lev: This mock should no longer be necessary once futures are enabled. class_mocker.patch( 'freqtrade.exchange.exchange.Exchange.validate_trading_mode_and_collateral') class_mocker.patch( From aabca85a5f7f4dc5b578670f25a26145ed84b221 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Dec 2021 19:50:58 +0100 Subject: [PATCH 3/8] Update `_calculate_funding_fees` to reuse existing async infrastructure --- freqtrade/exchange/exchange.py | 53 +++++++++++++++--------------- tests/exchange/test_ccxt_compat.py | 30 +++++++++++++---- tests/exchange/test_exchange.py | 10 +++--- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e09ed1c52..98014fa5f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1498,11 +1498,17 @@ class Exchange: params = deepcopy(self._ft_has.get('ohlcv_params', {})) if candle_type != CandleType.SPOT: params.update({'price': candle_type}) - data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, - since=since_ms, - limit=self.ohlcv_candle_limit(timeframe), - params=params) - + if candle_type != CandleType.FUNDING_RATE: + data = await self._api_async.fetch_ohlcv( + pair, timeframe=timeframe, since=since_ms, + limit=self.ohlcv_candle_limit(timeframe), params=params) + else: + # Funding rate + data = await self._api_async.fetch_funding_rate_history( + pair, since=since_ms, + limit=self.ohlcv_candle_limit(timeframe)) + # Convert funding rate to candle pattern + data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data] # Some exchanges sort OHLCV in ASC order and others in DESC. # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) # while GDAX returns the list of OHLCV in DESC order (newest first, oldest last) @@ -1882,28 +1888,23 @@ class Exchange: close_date = datetime.now(timezone.utc) open_timestamp = int(open_date.timestamp()) * 1000 # close_timestamp = int(close_date.timestamp()) * 1000 - funding_rate_history = self.get_funding_rate_history( - pair, - open_timestamp + + mark_comb: PairWithTimeframe = ( + pair, '1h', CandleType.from_string(self._ft_has["mark_ohlcv_price"])) + # TODO-lev: funding_rate downloading this way is not yet possible. + funding_comb: PairWithTimeframe = (pair, '1h', CandleType.FUNDING_RATE) + candle_histories = self.refresh_latest_ohlcv( + [mark_comb, funding_comb], + since_ms=open_timestamp, + cache=False, + drop_incomplete=False, ) - mark_price_history = self._get_mark_price_history( - pair, - open_timestamp - ) - for timestamp in funding_rate_history.keys(): - funding_rate = funding_rate_history[timestamp] - if timestamp in mark_price_history: - mark_price = mark_price_history[timestamp] - fees += self._get_funding_fee( - size=amount, - mark_price=mark_price, - funding_rate=funding_rate - ) - else: - logger.warning( - f"Mark price for {pair} at timestamp {timestamp} not found in " - f"funding_rate_history Funding fee calculation may be incorrect" - ) + funding_rates = candle_histories[funding_comb] + mark_rates = candle_histories[mark_comb] + + df = funding_rates.merge(mark_rates, on='date', how="inner", suffixes=["_fund", "_mark"]) + # TODO-lev: filter for relevant timeperiod? + fees = sum(df['open_fund'] * df['open_mark'] * amount) return fees diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index c9b2173b6..1da109cfb 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -195,7 +195,6 @@ class TestCCXTExchange(): assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) def test_ccxt_fetch_funding_rate_history(self, exchange_futures): - # TODO-lev: enable this test once Futures mode is enabled. exchange, exchangename = exchange_futures if not exchange: # exchange_futures only returns values for supported exchanges @@ -203,15 +202,21 @@ class TestCCXTExchange(): pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) since = int((datetime.now(timezone.utc) - timedelta(days=5)).timestamp() * 1000) + pair_tf = (pair, '1h', CandleType.FUNDING_RATE) - rate = exchange.get_funding_rate_history(pair, since) - assert isinstance(rate, dict) + funding_ohlcv = exchange.refresh_latest_ohlcv( + [pair_tf], + since_ms=since, + drop_incomplete=False) + + assert isinstance(funding_ohlcv, dict) + rate = funding_ohlcv[pair_tf] expected_tf = exchange._ft_has['mark_ohlcv_timeframe'] this_hour = timeframe_to_prev_date(expected_tf) - prev_tick = timeframe_to_prev_date(expected_tf, this_hour - timedelta(minutes=1)) - assert rate[int(this_hour.timestamp() * 1000)] != 0.0 - assert rate[int(prev_tick.timestamp() * 1000)] != 0.0 + prev_hour = timeframe_to_prev_date(expected_tf, this_hour - timedelta(minutes=1)) + assert rate[rate['date'] == this_hour].iloc[0]['open'] != 0.0 + assert rate[rate['date'] == prev_hour].iloc[0]['open'] != 0.0 def test_ccxt_fetch_mark_price_history(self, exchange_futures): exchange, exchangename = exchange_futures @@ -237,6 +242,19 @@ class TestCCXTExchange(): assert mark_candles[mark_candles['date'] == prev_hour].iloc[0]['open'] != 0.0 assert mark_candles[mark_candles['date'] == this_hour].iloc[0]['open'] != 0.0 + def test_ccxt__calculate_funding_fees(self, exchange_futures): + exchange, exchangename = exchange_futures + if not exchange: + # exchange_futures only returns values for supported exchanges + return + pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) + since = datetime.now(timezone.utc) - timedelta(days=5) + + funding_fee = exchange._calculate_funding_fees(pair, 20, open_date=since) + + assert isinstance(funding_fee, float) + # assert funding_fee > 0 + # TODO: tests fetch_trades (?) def test_ccxt_get_fee(self, exchange): diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 4632e6c56..23c0c6982 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3566,14 +3566,14 @@ def test__calculate_funding_fees( 'gateio': funding_rate_history_octohourly, }[exchange][rate_start:rate_end] api_mock = MagicMock() - api_mock.fetch_funding_rate_history = MagicMock(return_value=funding_rate_history) - api_mock.fetch_ohlcv = MagicMock(return_value=mark_ohlcv) + api_mock.fetch_funding_rate_history = get_mock_coro(return_value=funding_rate_history) + api_mock.fetch_ohlcv = get_mock_coro(return_value=mark_ohlcv) type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange) funding_fees = exchange._calculate_funding_fees('ADA/USDT', amount, d1, d2) - assert funding_fees == expected_fees + assert pytest.approx(funding_fees, expected_fees) @ pytest.mark.parametrize('exchange,expected_fees', [ @@ -3590,8 +3590,8 @@ def test__calculate_funding_fees_datetime_called( expected_fees ): api_mock = MagicMock() - api_mock.fetch_ohlcv = MagicMock(return_value=mark_ohlcv) - api_mock.fetch_funding_rate_history = MagicMock(return_value=funding_rate_history_octohourly) + api_mock.fetch_ohlcv = get_mock_coro(return_value=mark_ohlcv) + api_mock.fetch_funding_rate_history = get_mock_coro(return_value=funding_rate_history_octohourly) type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True}) From a87d2d62bb35e68b37bdbf4f475efd59b1beccfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Dec 2021 19:52:02 +0100 Subject: [PATCH 4/8] Remove no longer needed method get_funding_rate_history --- freqtrade/exchange/exchange.py | 36 ------------------------------- tests/exchange/test_exchange.py | 38 --------------------------------- 2 files changed, 74 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 98014fa5f..d491f89a2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1925,42 +1925,6 @@ class Exchange: else: return 0.0 - @retrier - def get_funding_rate_history(self, pair: str, since: int) -> Dict: - """ - :param pair: quote/base currency pair - :param since: timestamp in ms of the beginning time - :param end: timestamp in ms of the end time - """ - if not self.exchange_has("fetchFundingRateHistory"): - raise ExchangeError( - f"fetch_funding_rate_history is not available using {self.name}" - ) - - # TODO-lev: Gateio has a max limit into the past of 333 days, okex has a limit of 3 months - try: - funding_history: Dict = {} - response = self._api.fetch_funding_rate_history( - pair, - limit=1000, - since=since - ) - for fund in response: - d = datetime.fromtimestamp(int(fund['timestamp'] / 1000), timezone.utc) - # Round down to the nearest hour, in case of a delayed timestamp - # The millisecond timestamps can be delayed ~20ms - time = int(timeframe_to_prev_date('1h', d).timestamp() * 1000) - - funding_history[time] = fund['fundingRate'] - return funding_history - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: return exchange_name in ccxt_exchanges(ccxt_module) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 23c0c6982..8ccb0f67f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3445,44 +3445,6 @@ def test__get_mark_price_history(mocker, default_conf, mark_ohlcv): ) -def test_get_funding_rate_history(mocker, default_conf, funding_rate_history_hourly): - api_mock = MagicMock() - api_mock.fetch_funding_rate_history = MagicMock(return_value=funding_rate_history_hourly) - type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True}) - - # mocker.patch('freqtrade.exchange.Exchange.get_funding_fees', lambda pair, since: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - funding_rates = exchange.get_funding_rate_history('ADA/USDT', 1635580800001) - - assert funding_rates == { - 1630454400000: -0.000008, - 1630458000000: -0.000004, - 1630461600000: 0.000012, - 1630465200000: -0.000003, - 1630468800000: -0.000007, - 1630472400000: 0.000003, - 1630476000000: 0.000019, - 1630479600000: 0.000003, - 1630483200000: -0.000003, - 1630486800000: 0, - 1630490400000: 0.000013, - 1630494000000: 0.000077, - 1630497600000: 0.000072, - 1630501200000: 0.000097, - } - - ccxt_exceptionhandlers( - mocker, - default_conf, - api_mock, - "binance", - "get_funding_rate_history", - "fetch_funding_rate_history", - pair="ADA/USDT", - since=1630454400000 - ) - - @pytest.mark.parametrize('exchange,rate_start,rate_end,d1,d2,amount,expected_fees', [ ('binance', 0, 2, "2021-09-01 00:00:00", "2021-09-01 08:00:00", 30.0, -0.0009140999999999999), ('binance', 0, 2, "2021-09-01 00:00:15", "2021-09-01 08:00:00", 30.0, -0.0009140999999999999), From 6948414e476f3229e1c7899488ab625186751a2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Dec 2021 19:52:48 +0100 Subject: [PATCH 5/8] Remove no longer necessary method _get_mark_price_history --- freqtrade/exchange/exchange.py | 40 --------------------------------- tests/exchange/test_exchange.py | 40 ++------------------------------- 2 files changed, 2 insertions(+), 78 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d491f89a2..b97f9cc94 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1822,46 +1822,6 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e - @retrier - def _get_mark_price_history(self, pair: str, since: int) -> Dict: - """ - Get's the mark price history for a pair - :param pair: The quote/base pair of the trade - :param since: The earliest time to start downloading candles, in ms. - """ - - try: - candles = self._api.fetch_ohlcv( - pair, - timeframe="1h", - since=since, - params={ - 'price': self._ft_has["mark_ohlcv_price"] - } - ) - history = {} - for candle in candles: - d = datetime.fromtimestamp(int(candle[0] / 1000), timezone.utc) - # Round down to the nearest hour, in case of a delayed timestamp - # The millisecond timestamps can be delayed ~20ms - time = timeframe_to_prev_date('1h', d).timestamp() * 1000 - opening_mark_price = candle[1] - history[time] = opening_mark_price - return history - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical ' - f'mark price candle (OHLCV) data. Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError(f'Could not fetch historical mark price candle (OHLCV) data ' - f'for pair {pair} due to {e.__class__.__name__}. ' - f'Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch historical mark price candle (OHLCV) data ' - f'for pair {pair}. Message: {e}') from e - def _calculate_funding_fees( self, pair: str, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8ccb0f67f..54dc8689b 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3408,43 +3408,6 @@ def test__get_funding_fee( assert kraken._get_funding_fee(size, funding_rate, mark_price, time_in_ratio) == kraken_fee -def test__get_mark_price_history(mocker, default_conf, mark_ohlcv): - api_mock = MagicMock() - api_mock.fetch_ohlcv = MagicMock(return_value=mark_ohlcv) - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - - # mocker.patch('freqtrade.exchange.Exchange.get_funding_fees', lambda pair, since: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - mark_prices = exchange._get_mark_price_history("ADA/USDT", 1630454400000) - assert mark_prices == { - 1630454400000: 2.77, - 1630458000000: 2.73, - 1630461600000: 2.74, - 1630465200000: 2.76, - 1630468800000: 2.76, - 1630472400000: 2.77, - 1630476000000: 2.78, - 1630479600000: 2.78, - 1630483200000: 2.77, - 1630486800000: 2.77, - 1630490400000: 2.84, - 1630494000000: 2.81, - 1630497600000: 2.81, - 1630501200000: 2.82, - } - - ccxt_exceptionhandlers( - mocker, - default_conf, - api_mock, - "binance", - "_get_mark_price_history", - "fetch_ohlcv", - pair="ADA/USDT", - since=1635580800001 - ) - - @pytest.mark.parametrize('exchange,rate_start,rate_end,d1,d2,amount,expected_fees', [ ('binance', 0, 2, "2021-09-01 00:00:00", "2021-09-01 08:00:00", 30.0, -0.0009140999999999999), ('binance', 0, 2, "2021-09-01 00:00:15", "2021-09-01 08:00:00", 30.0, -0.0009140999999999999), @@ -3553,7 +3516,8 @@ def test__calculate_funding_fees_datetime_called( ): api_mock = MagicMock() api_mock.fetch_ohlcv = get_mock_coro(return_value=mark_ohlcv) - api_mock.fetch_funding_rate_history = get_mock_coro(return_value=funding_rate_history_octohourly) + api_mock.fetch_funding_rate_history = get_mock_coro( + return_value=funding_rate_history_octohourly) type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True}) From 17bd990053637e3015034f04637dddfb28eda2bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Dec 2021 09:49:48 +0100 Subject: [PATCH 6/8] Update funding_fee freqtradebot test --- freqtrade/exchange/exchange.py | 6 +- tests/exchange/test_exchange.py | 2 +- tests/test_freqtradebot.py | 131 ++++++++++++++++---------------- 3 files changed, 72 insertions(+), 67 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b97f9cc94..f05a0bd52 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1851,7 +1851,9 @@ class Exchange: mark_comb: PairWithTimeframe = ( pair, '1h', CandleType.from_string(self._ft_has["mark_ohlcv_price"])) - # TODO-lev: funding_rate downloading this way is not yet possible. + + # TODO-lev: 1h seems arbitrary and generates a lot of "empty" lines + # TODO-lev: probably a exchange-adjusted parameter would make more sense funding_comb: PairWithTimeframe = (pair, '1h', CandleType.FUNDING_RATE) candle_histories = self.refresh_latest_ohlcv( [mark_comb, funding_comb], @@ -1863,7 +1865,7 @@ class Exchange: mark_rates = candle_histories[mark_comb] df = funding_rates.merge(mark_rates, on='date', how="inner", suffixes=["_fund", "_mark"]) - # TODO-lev: filter for relevant timeperiod? + df = df[(df['date'] >= open_date) & (df['date'] <= close_date)] fees = sum(df['open_fund'] * df['open_mark'] * amount) return fees diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 54dc8689b..7fe666565 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3498,7 +3498,7 @@ def test__calculate_funding_fees( exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange) funding_fees = exchange._calculate_funding_fees('ADA/USDT', amount, d1, d2) - assert pytest.approx(funding_fees, expected_fees) + assert pytest.approx(funding_fees) == expected_fees @ pytest.mark.parametrize('exchange,expected_fees', [ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 7c22078e2..1a1384442 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -9,6 +9,7 @@ from unittest.mock import ANY, MagicMock, PropertyMock import arrow import pytest +from pandas import DataFrame from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT from freqtrade.enums import CandleType, RPCMessageType, RunMode, SellType, SignalDirection, State @@ -4802,60 +4803,67 @@ def test_update_funding_fees( patch_exchange(mocker) default_conf['trading_mode'] = 'futures' default_conf['collateral'] = 'isolated' - default_conf['dry_run'] = True - timestamp_midnight = 1630454400000 - timestamp_eight = 1630483200000 - funding_rates_midnight = { - "LTC/BTC": { - timestamp_midnight: 0.00032583, - }, - "ETH/BTC": { - timestamp_midnight: 0.0001, - }, - "XRP/BTC": { - timestamp_midnight: 0.00049426, - } - } - funding_rates_eight = { - "LTC/BTC": { - timestamp_midnight: 0.00032583, - timestamp_eight: 0.00024472, - }, - "ETH/BTC": { - timestamp_midnight: 0.0001, - timestamp_eight: 0.0001, - }, - "XRP/BTC": { - timestamp_midnight: 0.00049426, - timestamp_eight: 0.00032715, - } + date_midnight = arrow.get('2021-09-01 00:00:00') + date_eight = arrow.get('2021-09-01 08:00:00') + date_sixteen = arrow.get('2021-09-01 16:00:00') + columns = ['date', 'open', 'high', 'low', 'close', 'volume'] + # 16:00 entry is actually never used + # But should be kept in the test to ensure we're filtering correctly. + funding_rates = { + "LTC/BTC": + DataFrame([ + [date_midnight, 0.00032583, 0, 0, 0, 0], + [date_eight, 0.00024472, 0, 0, 0, 0], + [date_sixteen, 0.00024472, 0, 0, 0, 0], + ], columns=columns), + "ETH/BTC": + DataFrame([ + [date_midnight, 0.0001, 0, 0, 0, 0], + [date_eight, 0.0001, 0, 0, 0, 0], + [date_sixteen, 0.0001, 0, 0, 0, 0], + ], columns=columns), + "XRP/BTC": + DataFrame([ + [date_midnight, 0.00049426, 0, 0, 0, 0], + [date_eight, 0.00032715, 0, 0, 0, 0], + [date_sixteen, 0.00032715, 0, 0, 0, 0], + ], columns=columns) } mark_prices = { - "LTC/BTC": { - timestamp_midnight: 3.3, - timestamp_eight: 3.2, - }, - "ETH/BTC": { - timestamp_midnight: 2.4, - timestamp_eight: 2.5, - }, - "XRP/BTC": { - timestamp_midnight: 1.2, - timestamp_eight: 1.2, - } + "LTC/BTC": + DataFrame([ + [date_midnight, 3.3, 0, 0, 0, 0], + [date_eight, 3.2, 0, 0, 0, 0], + [date_sixteen, 3.2, 0, 0, 0, 0], + ], columns=columns), + "ETH/BTC": + DataFrame([ + [date_midnight, 2.4, 0, 0, 0, 0], + [date_eight, 2.5, 0, 0, 0, 0], + [date_sixteen, 2.5, 0, 0, 0, 0], + ], columns=columns), + "XRP/BTC": + DataFrame([ + [date_midnight, 1.2, 0, 0, 0, 0], + [date_eight, 1.2, 0, 0, 0, 0], + [date_sixteen, 1.2, 0, 0, 0, 0], + ], columns=columns) } - mocker.patch( - 'freqtrade.exchange.Exchange._get_mark_price_history', - side_effect=lambda pair, since: mark_prices[pair] - ) + def refresh_latest_ohlcv_mock(pairlist, **kwargs): + ret = {} + for p, tf, ct in pairlist: + if ct == CandleType.MARK: + ret[(p, tf, ct)] = mark_prices[p] + else: + ret[(p, tf, ct)] = funding_rates[p] - mocker.patch( - 'freqtrade.exchange.Exchange.get_funding_rate_history', - side_effect=lambda pair, since: funding_rates_midnight[pair] - ) + return ret + + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', + side_effect=refresh_latest_ohlcv_mock) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -4880,38 +4888,33 @@ def test_update_funding_fees( trades = Trade.get_open_trades() assert len(trades) == 3 for trade in trades: - assert trade.funding_fees == ( + assert pytest.approx(trade.funding_fees) == ( trade.amount * - mark_prices[trade.pair][timestamp_midnight] * - funding_rates_midnight[trade.pair][timestamp_midnight] + mark_prices[trade.pair].iloc[0]['open'] * + funding_rates[trade.pair].iloc[0]['open'] ) mocker.patch('freqtrade.exchange.Exchange.create_order', return_value=open_exit_order) - # create_mock_trades(fee, False) time_machine.move_to("2021-09-01 08:00:00 +00:00") - mocker.patch( - 'freqtrade.exchange.Exchange.get_funding_rate_history', - side_effect=lambda pair, since: funding_rates_eight[pair] - ) if schedule_off: for trade in trades: - assert trade.funding_fees == ( - trade.amount * - mark_prices[trade.pair][timestamp_midnight] * - funding_rates_eight[trade.pair][timestamp_midnight] - ) freqtrade.execute_trade_exit( trade=trade, # The values of the next 2 params are irrelevant for this test limit=ticker_usdt_sell_up()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.ROI) ) + assert trade.funding_fees == pytest.approx(sum( + trade.amount * + mark_prices[trade.pair].iloc[0:2]['open'] * + funding_rates[trade.pair].iloc[0:2]['open'] + )) + else: freqtrade._schedule.run_pending() # Funding fees for 00:00 and 08:00 for trade in trades: - assert trade.funding_fees == sum([ + assert trade.funding_fees == pytest.approx(sum( trade.amount * - mark_prices[trade.pair][time] * - funding_rates_eight[trade.pair][time] for time in mark_prices[trade.pair].keys() - ]) + mark_prices[trade.pair].iloc[0:2]['open'] * funding_rates[trade.pair].iloc[0:2]['open'] + )) From ddce28c12db4a6b8ca0b50c564f3578feddcc074 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Dec 2021 15:32:02 +0100 Subject: [PATCH 7/8] Update data downloading to include funding_fee downloads --- freqtrade/data/history/history_utils.py | 25 ++++++++++++++----------- freqtrade/exchange/exchange.py | 10 ++++------ tests/data/test_history.py | 2 +- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 64297c7e5..57fdc4a14 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -283,17 +283,20 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. timeframe = exchange._ft_has['mark_ohlcv_timeframe'] - candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) - - # TODO: this could be in most parts to the above. - if erase: - if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): - logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') - _download_pair_history(pair=pair, process=process, - datadir=datadir, exchange=exchange, - timerange=timerange, data_handler=data_handler, - timeframe=str(timeframe), new_pairs_days=new_pairs_days, - candle_type=candle_type) + fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) + # All exchanges need FundingRate for futures trading. + # The timeframe is aligned to the mark-price timeframe. + for candle_type in (CandleType.FUNDING_RATE, fr_candle_type): + # TODO: this could be in most parts to the above. + if erase: + if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): + logger.info( + f'Deleting existing data for pair {pair}, interval {timeframe}.') + _download_pair_history(pair=pair, process=process, + datadir=datadir, exchange=exchange, + timerange=timerange, data_handler=data_handler, + timeframe=str(timeframe), new_pairs_days=new_pairs_days, + candle_type=candle_type) return pairs_not_available diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index f05a0bd52..b6709d0db 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1840,8 +1840,8 @@ class Exchange: if self.funding_fee_cutoff(open_date): open_date += timedelta(hours=1) - - open_date = timeframe_to_prev_date('1h', open_date) + timeframe = self._ft_has['mark_ohlcv_timeframe'] + open_date = timeframe_to_prev_date(timeframe, open_date) fees: float = 0 if not close_date: @@ -1850,11 +1850,9 @@ class Exchange: # close_timestamp = int(close_date.timestamp()) * 1000 mark_comb: PairWithTimeframe = ( - pair, '1h', CandleType.from_string(self._ft_has["mark_ohlcv_price"])) + pair, timeframe, CandleType.from_string(self._ft_has["mark_ohlcv_price"])) - # TODO-lev: 1h seems arbitrary and generates a lot of "empty" lines - # TODO-lev: probably a exchange-adjusted parameter would make more sense - funding_comb: PairWithTimeframe = (pair, '1h', CandleType.FUNDING_RATE) + funding_comb: PairWithTimeframe = (pair, timeframe, CandleType.FUNDING_RATE) candle_histories = self.refresh_latest_ohlcv( [mark_comb, funding_comb], since_ms=open_timestamp, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 678a0b31b..d70d69080 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -490,7 +490,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No @pytest.mark.parametrize('trademode,callcount', [ ('spot', 4), ('margin', 4), - ('futures', 6), + ('futures', 8), # Called 8 times - 4 normal, 2 funding and 2 mark/index calls ]) def test_refresh_backtest_ohlcv_data( mocker, default_conf, markets, caplog, testdatadir, trademode, callcount): From a557451eee75499925b5ac68ce501b669e047bbe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Dec 2021 14:48:59 +0100 Subject: [PATCH 8/8] Okex uses 4h mark candle timeframe --- freqtrade/exchange/exchange.py | 4 +++- freqtrade/exchange/okex.py | 2 ++ tests/exchange/test_ccxt_compat.py | 9 +++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b6709d0db..0817df0fc 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1841,6 +1841,8 @@ class Exchange: if self.funding_fee_cutoff(open_date): open_date += timedelta(hours=1) timeframe = self._ft_has['mark_ohlcv_timeframe'] + timeframe_ff = self._ft_has.get('funding_fee_timeframe', + self._ft_has['mark_ohlcv_timeframe']) open_date = timeframe_to_prev_date(timeframe, open_date) fees: float = 0 @@ -1852,7 +1854,7 @@ class Exchange: mark_comb: PairWithTimeframe = ( pair, timeframe, CandleType.from_string(self._ft_has["mark_ohlcv_price"])) - funding_comb: PairWithTimeframe = (pair, timeframe, CandleType.FUNDING_RATE) + funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE) candle_histories = self.refresh_latest_ohlcv( [mark_comb, funding_comb], since_ms=open_timestamp, diff --git a/freqtrade/exchange/okex.py b/freqtrade/exchange/okex.py index 62e6d977b..20e142824 100644 --- a/freqtrade/exchange/okex.py +++ b/freqtrade/exchange/okex.py @@ -16,6 +16,8 @@ class Okex(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 100, + "mark_ohlcv_timeframe": "4h", + "funding_fee_timeframe": "8h", } _supported_trading_mode_collateral_pairs: List[Tuple[TradingMode, Collateral]] = [ diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 1da109cfb..122e6e37c 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -202,7 +202,9 @@ class TestCCXTExchange(): pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) since = int((datetime.now(timezone.utc) - timedelta(days=5)).timestamp() * 1000) - pair_tf = (pair, '1h', CandleType.FUNDING_RATE) + timeframe_ff = exchange._ft_has.get('funding_fee_timeframe', + exchange._ft_has['mark_ohlcv_timeframe']) + pair_tf = (pair, timeframe_ff, CandleType.FUNDING_RATE) funding_ohlcv = exchange.refresh_latest_ohlcv( [pair_tf], @@ -212,9 +214,8 @@ class TestCCXTExchange(): assert isinstance(funding_ohlcv, dict) rate = funding_ohlcv[pair_tf] - expected_tf = exchange._ft_has['mark_ohlcv_timeframe'] - this_hour = timeframe_to_prev_date(expected_tf) - prev_hour = timeframe_to_prev_date(expected_tf, this_hour - timedelta(minutes=1)) + this_hour = timeframe_to_prev_date(timeframe_ff) + prev_hour = timeframe_to_prev_date(timeframe_ff, this_hour - timedelta(minutes=1)) assert rate[rate['date'] == this_hour].iloc[0]['open'] != 0.0 assert rate[rate['date'] == prev_hour].iloc[0]['open'] != 0.0