From cce8d1aa4d4f424d35a97c6f7cbb79a698dbd38b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 May 2022 08:48:34 +0000 Subject: [PATCH 01/35] Update get_market_leverage_tiers to be async --- freqtrade/exchange/exchange.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c1a9059a7..c292d7dcb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2131,10 +2131,11 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e - @retrier - def get_market_leverage_tiers(self, symbol) -> List[Dict]: + @retrier_async + async def get_market_leverage_tiers(self, symbol: str) -> Tuple[str, List[Dict]]: try: - return self._api.fetch_market_leverage_tiers(symbol) + tier = await self._api_async.fetch_market_leverage_tiers(symbol) + return symbol, tier except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: @@ -2168,8 +2169,14 @@ class Exchange: f"Initializing leverage_tiers for {len(symbols)} markets. " "This will take about a minute.") - for symbol in sorted(symbols): - tiers[symbol] = self.get_market_leverage_tiers(symbol) + coros = [self.get_market_leverage_tiers(symbol) for symbol in sorted(symbols)] + + for input_coro in chunks(coros, 100): + + results = self.loop.run_until_complete( + asyncio.gather(*input_coro, return_exceptions=True)) + for symbol, res in results: + tiers[symbol] = res logger.info(f"Done initializing {len(symbols)} markets.") From ea537b32c73f6ce0a0cd17c1630292a3f0bcb9c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 May 2022 11:40:14 +0000 Subject: [PATCH 02/35] Update tests for leverage_tier_loading --- tests/conftest.py | 16 ++++++++++++++-- tests/exchange/test_okx.py | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 02738b0e9..c5c253891 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,9 +78,21 @@ def get_args(args): # Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines -def get_mock_coro(return_value): +# TODO: This should be replaced with AsyncMock once support for python 3.7 is dropped. +def get_mock_coro(return_value=None, side_effect=None): async def mock_coro(*args, **kwargs): - return return_value + if side_effect: + if isinstance(side_effect, list): + effect = side_effect.pop(0) + else: + effect = side_effect + if isinstance(effect, Exception): + raise effect + if callable(effect): + return effect(*args, **kwargs) + return effect + else: + return return_value return Mock(wraps=mock_coro) diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 19c09ad9e..91c4a3368 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -6,7 +6,7 @@ import pytest from freqtrade.enums import MarginMode, TradingMode from freqtrade.enums.candletype import CandleType from freqtrade.exchange.exchange import timeframe_to_minutes -from tests.conftest import get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -273,7 +273,7 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets): 'fetchLeverageTiers': False, 'fetchMarketLeverageTiers': True, }) - api_mock.fetch_market_leverage_tiers = MagicMock(side_effect=[ + api_mock.fetch_market_leverage_tiers = get_mock_coro(side_effect=[ [ { 'tier': 1, From f9e2e87346319bb32e596538a91340b8b4474b09 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 20:03:36 +0200 Subject: [PATCH 03/35] Improve some formatting and typehints --- freqtrade/rpc/rpc.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index da5144dab..8b1cdb851 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -512,7 +512,7 @@ class RPC: def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: """ Returns current account balance per crypto """ - currencies = [] + currencies: List[Dict] = [] total = 0.0 try: tickers = self._freqtrade.exchange.get_tickers(cached=True) @@ -547,13 +547,12 @@ class RPC: except (ExchangeError): logger.warning(f" Could not get rate for pair {coin}.") continue - total = total + (est_stake or 0) + total = total + est_stake currencies.append({ 'currency': coin, - # TODO: The below can be simplified if we don't assign None to values. - 'free': balance.free if balance.free is not None else 0, - 'balance': balance.total if balance.total is not None else 0, - 'used': balance.used if balance.used is not None else 0, + 'free': balance.free, + 'balance': balance.total, + 'used': balance.used, 'est_stake': est_stake or 0, 'stake': stake_currency, 'side': 'long', @@ -583,7 +582,6 @@ class RPC: total, stake_currency, fiat_display_currency) if self._fiat_converter else 0 trade_count = len(Trade.get_trades_proxy()) - starting_capital_ratio = 0.0 starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0 starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0 @@ -871,7 +869,7 @@ class RPC: else: errors[pair] = { 'error_msg': f"Pair {pair} is not in the current blacklist." - } + } resp = self._rpc_blacklist() resp['errors'] = errors return resp From 8f32fa5cb30efed0fa4f3d81e676951b5554dd8a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 20:13:07 +0200 Subject: [PATCH 04/35] Avoid exception on exchange recycling if __init__ fails --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c1a9059a7..465cce300 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -93,7 +93,7 @@ class Exchange: :return: None """ self._api: ccxt.Exchange - self._api_async: ccxt_async.Exchange + self._api_async: ccxt_async.Exchange = None self._markets: Dict = {} self._trading_fees: Dict[str, Any] = {} self._leverage_tiers: Dict[str, List[Dict]] = {} From 14a859c190a1fa8b89a13b8d756127546b822ff5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Jun 2022 19:50:13 +0200 Subject: [PATCH 05/35] Improve some documentation around futures / leverage --- docs/leverage.md | 15 ++++++++++++++- docs/stoploss.md | 13 +++++++++++++ docs/strategy-callbacks.md | 3 +++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/leverage.md b/docs/leverage.md index 2ee6f8444..491e6eda0 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -64,7 +64,10 @@ You will also have to pick a "margin mode" (explanation below) - with freqtrade ### Margin mode -The possible values are: `isolated`, or `cross`(*currently unavailable*) +On top of `trading_mode` - you will also have to configure your `margin_mode`. +While freqtrade currently only supports one margin mode, this will change, and by configuring it now you're all set for future updates. + +The possible values are: `isolated`, or `cross`(*currently unavailable*). #### Isolated margin mode @@ -82,6 +85,16 @@ One account is used to share collateral between markets (trading pairs). Margin "margin_mode": "cross" ``` +## Set leverage to use + +Different strategies and risk profiles will require different levels of leverage. +While you could configure one static leverage value - freqtrade offers you the flexibility to adjust this via [strategy leverage callback](strategy-callbacks.md#leverage-callback) - which allows you to use different leverages by pair, or based on some other factor benefitting your strategy result. + +If not implemented, leverage defaults to 1x (no leverage). + +!!! Warning + Higher leverage also equals higher risk - be sure you fully understand the implications of using leverage! + ## Understand `liquidation_buffer` *Defaults to `0.05`* diff --git a/docs/stoploss.md b/docs/stoploss.md index 573fdbd6c..83f787947 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -191,6 +191,19 @@ For example, simplified math: !!! Tip Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade. +## Stoploss and Leverage + +Stoploss should be thought of as "risk on this trade" - so a stoploss of 10% on a 100$ trade means you are willing to lose 10$ (10%) on this trade - which would trigger if the price moves 10% to the downside. + +When using leverage, the same principle is applied - with stoploss defining the risk on the trade (the amount you are willing to lose). + +Therefore, a stoploss of 10% on a 10x trade would trigger on a 1% price move. +If your stake amount (own capital) was 100$ - this trade would be 1000$ at 10x (after leverage). +If price moves 1% - you've lost 10$ of your own capital - therfore stoploss will trigger in this case. + +Make sure to be aware of this, and avoid using too tight stoploss (at 10x leverage, 10% risk may be too little to allow the trade to "breath" a little). + + ## Changing stoploss on open trades A stoploss on an open trade can be changed by changing the value in the configuration or strategy and use the `/reload_config` command (alternatively, completely stopping and restarting the bot also works). diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 410641f44..beffba56b 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -823,3 +823,6 @@ class AwesomeStrategy(IStrategy): """ return 1.0 ``` + +All profit calculations include leverage. Stoploss / ROI also include leverage in their calculation. +Defining a stoploss of 10% at 10x leverage would trigger the stoploss with a 1% move to the downside. From 575b4ead1a04bffc4c3064abcfe85fe77574b7c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 06:29:17 +0000 Subject: [PATCH 06/35] Update Test with funding_fee 0 --- tests/test_persistence.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index be19a3f5f..836b17a55 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -884,6 +884,17 @@ def test_calc_close_trade_price( ('binance', False, 3, 2.2, 0.0025, 4.684999, 0.23366583, futures, -1), ('binance', True, 1, 2.2, 0.0025, -7.315, -0.12222222, futures, -1), ('binance', True, 3, 2.2, 0.0025, -7.315, -0.36666666, futures, -1), + + # FUTURES, funding_fee=0 + ('binance', False, 1, 2.1, 0.0025, 2.6925, 0.04476309, futures, 0), + ('binance', False, 3, 2.1, 0.0025, 2.6925, 0.13428928, futures, 0), + ('binance', True, 1, 2.1, 0.0025, -3.3074999, -0.05526316, futures, 0), + ('binance', True, 3, 2.1, 0.0025, -3.3074999, -0.16578947, futures, 0), + + ('binance', False, 1, 1.9, 0.0025, -3.2925, -0.05473815, futures, 0), + ('binance', False, 3, 1.9, 0.0025, -3.2925, -0.16421446, futures, 0), + ('binance', True, 1, 1.9, 0.0025, 2.7075, 0.0452381, futures, 0), + ('binance', True, 3, 1.9, 0.0025, 2.7075, 0.13571429, futures, 0), ]) @pytest.mark.usefixtures("init_persistence") def test_calc_profit( From 76cae8e8e3b2710983084acbfa6a0e88fa0e2c81 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 06:53:40 +0000 Subject: [PATCH 07/35] Update tests to always provide rate to profit calculations --- tests/rpc/test_rpc_apiserver.py | 8 ++++---- tests/test_freqtradebot.py | 2 +- tests/test_persistence.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8b3ac18ac..ada1a82ec 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -852,8 +852,8 @@ def test_api_performance(botclient, fee): close_rate=0.265441, ) - trade.close_profit = trade.calc_profit_ratio() - trade.close_profit_abs = trade.calc_profit() + trade.close_profit = trade.calc_profit_ratio(trade.close_rate) + trade.close_profit_abs = trade.calc_profit(trade.close_rate) Trade.query.session.add(trade) trade = Trade( @@ -868,8 +868,8 @@ def test_api_performance(botclient, fee): fee_open=fee.return_value, close_rate=0.391 ) - trade.close_profit = trade.calc_profit_ratio() - trade.close_profit_abs = trade.calc_profit() + trade.close_profit = trade.calc_profit_ratio(trade.close_rate) + trade.close_profit_abs = trade.calc_profit(trade.close_rate) Trade.query.session.add(trade) Trade.commit() diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3fd16f925..4f3d5f667 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2151,7 +2151,7 @@ def test_handle_trade( assert trade.close_rate == 2.0 if is_short else 2.2 assert trade.close_profit == close_profit - assert trade.calc_profit() == 5.685 + assert trade.calc_profit(trade.close_rate) == 5.685 assert trade.close_date is not None assert trade.exit_reason == 'sell_signal1' diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 836b17a55..de250e3e6 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -606,9 +606,9 @@ def test_calc_open_close_trade_price( trade.close_rate = 2.2 trade.recalc_open_trade_value() assert isclose(trade._calc_open_trade_value(), open_value) - assert isclose(trade.calc_close_trade_value(), close_value) - assert isclose(trade.calc_profit(), round(profit, 8)) - assert pytest.approx(trade.calc_profit_ratio()) == profit_ratio + assert isclose(trade.calc_close_trade_value(trade.close_rate), close_value) + assert isclose(trade.calc_profit(trade.close_rate), round(profit, 8)) + assert pytest.approx(trade.calc_profit_ratio(trade.close_rate)) == profit_ratio @pytest.mark.usefixtures("init_persistence") @@ -660,7 +660,7 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee): trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') trade.update_trade(oobj) - assert trade.calc_close_trade_value() == 0.0 + assert trade.calc_close_trade_value(trade.close_rate) == 0.0 @pytest.mark.usefixtures("init_persistence") From d7770c507b4e655a74226eada048d86fe7d6fdb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 07:00:42 +0000 Subject: [PATCH 08/35] Remove implicit use of certain rates in profit calculations --- freqtrade/persistence/trade_model.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3222a57b8..5a89849dd 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -624,8 +624,8 @@ class LocalTrade(): """ self.close_rate = rate self.close_date = self.close_date or datetime.utcnow() - self.close_profit = self.calc_profit_ratio() - self.close_profit_abs = self.calc_profit() + self.close_profit = self.calc_profit_ratio(rate) + self.close_profit_abs = self.calc_profit(rate) self.is_open = False self.exit_order_status = 'closed' self.open_order_id = None @@ -714,10 +714,10 @@ class LocalTrade(): return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) - def _calc_base_close(self, amount: Decimal, rate: Optional[float] = None, + def _calc_base_close(self, amount: Decimal, rate: float, fee: Optional[float] = None) -> Decimal: - close_trade = Decimal(amount) * Decimal(rate or self.close_rate) # type: ignore + close_trade = Decimal(amount) * Decimal(rate) fees = close_trade * Decimal(fee or self.fee_close) if self.is_short: @@ -725,15 +725,14 @@ class LocalTrade(): else: return close_trade - fees - def calc_close_trade_value(self, rate: Optional[float] = None, + def calc_close_trade_value(self, rate: float, fee: Optional[float] = None, interest_rate: Optional[float] = None) -> float: """ Calculate the close_rate including fee + :param rate: rate to compare with. :param fee: fee to use on the close rate (optional). If rate is not set self.fee will be used - :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used :param interest_rate: interest_charge for borrowing this coin (optional). If interest_rate is not set self.interest_rate will be used :return: Price in BTC of the open trade @@ -770,21 +769,20 @@ class LocalTrade(): raise OperationalException( f"{self.trading_mode.value} trading is not yet available using freqtrade") - def calc_profit(self, rate: Optional[float] = None, + def calc_profit(self, rate: float, fee: Optional[float] = None, interest_rate: Optional[float] = None) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade + :param rate: close rate to compare with. :param fee: fee to use on the close rate (optional). If fee is not set self.fee will be used - :param rate: close rate to compare with (optional). - If rate is not set self.close_rate will be used :param interest_rate: interest_charge for borrowing this coin (optional). If interest_rate is not set self.interest_rate will be used :return: profit in stake currency as float """ close_trade_value = self.calc_close_trade_value( - rate=(rate or self.close_rate), + rate=rate, fee=(fee or self.fee_close), interest_rate=(interest_rate or self.interest_rate) ) @@ -795,20 +793,19 @@ class LocalTrade(): profit = close_trade_value - self.open_trade_value return float(f"{profit:.8f}") - def calc_profit_ratio(self, rate: Optional[float] = None, + def calc_profit_ratio(self, rate: float, fee: Optional[float] = None, interest_rate: Optional[float] = None) -> float: """ Calculates the profit as ratio (including fee). - :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used + :param rate: rate to compare with. :param fee: fee to use on the close rate (optional). :param interest_rate: interest_charge for borrowing this coin (optional). If interest_rate is not set self.interest_rate will be used :return: profit ratio as float """ close_trade_value = self.calc_close_trade_value( - rate=(rate or self.close_rate), + rate=rate, fee=(fee or self.fee_close), interest_rate=(interest_rate or self.interest_rate) ) From 91f9818ae3b1c95ca2547916d7ea70be6c2a84a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 09:53:29 +0000 Subject: [PATCH 09/35] Simplify trade calculations --- freqtrade/persistence/trade_model.py | 43 +++++++++------------------- tests/test_persistence.py | 2 +- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 5a89849dd..3ac64ba6b 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -693,10 +693,9 @@ class LocalTrade(): """ self.open_trade_value = self._calc_open_trade_value() - def calculate_interest(self, interest_rate: Optional[float] = None) -> Decimal: + def calculate_interest(self) -> Decimal: """ - :param interest_rate: interest_charge for borrowing this coin(optional). - If interest_rate is not set self.interest_rate will be used + Calculate interest for this trade. Only applicable for Margin trading. """ zero = Decimal(0.0) # If nothing was borrowed @@ -709,7 +708,7 @@ class LocalTrade(): total_seconds = Decimal((now - open_date).total_seconds()) hours = total_seconds / sec_per_hour or zero - rate = Decimal(interest_rate or self.interest_rate) + rate = Decimal(self.interest_rate) borrowed = Decimal(self.borrowed) return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) @@ -726,16 +725,13 @@ class LocalTrade(): return close_trade - fees def calc_close_trade_value(self, rate: float, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: + fee: Optional[float] = None) -> float: """ - Calculate the close_rate including fee + Calculate the Trade's close value including fees :param rate: rate to compare with. :param fee: fee to use on the close rate (optional). - If rate is not set self.fee will be used - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used - :return: Price in BTC of the open trade + If rate is not set self.close_fee will be used + :return: value in stake currency of the open trade """ if rate is None and not self.close_rate: return 0.0 @@ -748,7 +744,7 @@ class LocalTrade(): elif (trading_mode == TradingMode.MARGIN): - total_interest = self.calculate_interest(interest_rate) + total_interest = self.calculate_interest() if self.is_short: amount = amount + total_interest @@ -769,22 +765,15 @@ class LocalTrade(): raise OperationalException( f"{self.trading_mode.value} trading is not yet available using freqtrade") - def calc_profit(self, rate: float, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: + def calc_profit(self, rate: float) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade :param rate: close rate to compare with. - :param fee: fee to use on the close rate (optional). - If fee is not set self.fee will be used - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used - :return: profit in stake currency as float + :return: profit in stake currency as float """ close_trade_value = self.calc_close_trade_value( rate=rate, - fee=(fee or self.fee_close), - interest_rate=(interest_rate or self.interest_rate) + fee=self.fee_close ) if self.is_short: @@ -793,21 +782,15 @@ class LocalTrade(): profit = close_trade_value - self.open_trade_value return float(f"{profit:.8f}") - def calc_profit_ratio(self, rate: float, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: + def calc_profit_ratio(self, rate: float) -> float: """ Calculates the profit as ratio (including fee). :param rate: rate to compare with. - :param fee: fee to use on the close rate (optional). - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used :return: profit ratio as float """ close_trade_value = self.calc_close_trade_value( rate=rate, - fee=(fee or self.fee_close), - interest_rate=(interest_rate or self.interest_rate) + fee=self.fee_close ) short_close_zero = (self.is_short and close_trade_value == 0.0) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index de250e3e6..8c12d2ea0 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -813,7 +813,7 @@ def test_calc_close_trade_price( funding_fees=funding_fees ) trade.open_order_id = 'close_trade' - assert round(trade.calc_close_trade_value(rate=close_rate, fee=fee_rate), 8) == result + assert round(trade.calc_close_trade_value(rate=close_rate), 8) == result @pytest.mark.parametrize( From 6bdf9c2a94a53820dc658d6b006f194e16b6e7f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 11:17:05 +0000 Subject: [PATCH 10/35] Simplify trade profit calculations further --- freqtrade/persistence/trade_model.py | 32 ++++++++++------------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3ac64ba6b..eb405942a 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -713,24 +713,20 @@ class LocalTrade(): return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) - def _calc_base_close(self, amount: Decimal, rate: float, - fee: Optional[float] = None) -> Decimal: + def _calc_base_close(self, amount: Decimal, rate: float, fee: float) -> Decimal: - close_trade = Decimal(amount) * Decimal(rate) - fees = close_trade * Decimal(fee or self.fee_close) + close_trade = amount * Decimal(rate) + fees = close_trade * Decimal(fee) if self.is_short: return close_trade + fees else: return close_trade - fees - def calc_close_trade_value(self, rate: float, - fee: Optional[float] = None) -> float: + def calc_close_trade_value(self, rate: float) -> float: """ Calculate the Trade's close value including fees :param rate: rate to compare with. - :param fee: fee to use on the close rate (optional). - If rate is not set self.close_fee will be used :return: value in stake currency of the open trade """ if rate is None and not self.close_rate: @@ -740,7 +736,7 @@ class LocalTrade(): trading_mode = self.trading_mode or TradingMode.SPOT if trading_mode == TradingMode.SPOT: - return float(self._calc_base_close(amount, rate, fee)) + return float(self._calc_base_close(amount, rate, self.fee_close)) elif (trading_mode == TradingMode.MARGIN): @@ -748,19 +744,19 @@ class LocalTrade(): if self.is_short: amount = amount + total_interest - return float(self._calc_base_close(amount, rate, fee)) + return float(self._calc_base_close(amount, rate, self.fee_close)) else: # Currency already owned for longs, no need to purchase - return float(self._calc_base_close(amount, rate, fee) - total_interest) + return float(self._calc_base_close(amount, rate, self.fee_close) - total_interest) elif (trading_mode == TradingMode.FUTURES): funding_fees = self.funding_fees or 0.0 # Positive funding_fees -> Trade has gained from fees. # Negative funding_fees -> Trade had to pay the fees. if self.is_short: - return float(self._calc_base_close(amount, rate, fee)) - funding_fees + return float(self._calc_base_close(amount, rate, self.fee_close)) - funding_fees else: - return float(self._calc_base_close(amount, rate, fee)) + funding_fees + return float(self._calc_base_close(amount, rate, self.fee_close)) + funding_fees else: raise OperationalException( f"{self.trading_mode.value} trading is not yet available using freqtrade") @@ -771,10 +767,7 @@ class LocalTrade(): :param rate: close rate to compare with. :return: profit in stake currency as float """ - close_trade_value = self.calc_close_trade_value( - rate=rate, - fee=self.fee_close - ) + close_trade_value = self.calc_close_trade_value(rate) if self.is_short: profit = self.open_trade_value - close_trade_value @@ -788,10 +781,7 @@ class LocalTrade(): :param rate: rate to compare with. :return: profit ratio as float """ - close_trade_value = self.calc_close_trade_value( - rate=rate, - fee=self.fee_close - ) + close_trade_value = self.calc_close_trade_value(rate) short_close_zero = (self.is_short and close_trade_value == 0.0) long_close_zero = (not self.is_short and self.open_trade_value == 0.0) From fda8248d41bce429415cb20292550df6d7e5e654 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 22:43:24 +0200 Subject: [PATCH 11/35] Gateio allow market orders on futures markets --- freqtrade/exchange/gateio.py | 1 - tests/exchange/test_gateio.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index 4147e8290..f69a0dc02 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -40,7 +40,6 @@ class Gateio(Exchange): ] def validate_ordertypes(self, order_types: Dict) -> None: - super().validate_ordertypes(order_types) if self.trading_mode != TradingMode.FUTURES: if any(v == 'market' for k, v in order_types.items()): diff --git a/tests/exchange/test_gateio.py b/tests/exchange/test_gateio.py index 92f8186a6..cbd4776fb 100644 --- a/tests/exchange/test_gateio.py +++ b/tests/exchange/test_gateio.py @@ -33,6 +33,12 @@ def test_validate_order_types_gateio(default_conf, mocker): match=r'Exchange .* does not support market orders.'): ExchangeResolver.load_exchange('gateio', default_conf, True) + # market-orders supported on futures markets. + default_conf['trading_mode'] = 'futures' + default_conf['margin_mode'] = 'isolated' + ex = ExchangeResolver.load_exchange('gateio', default_conf, True) + assert ex + @pytest.mark.usefixtures("init_persistence") def test_fetch_stoploss_order_gateio(default_conf, mocker): From 616bf315cbf8647f5f9114aee74e648f747b9a27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 23:02:39 +0200 Subject: [PATCH 12/35] gateio: futures market orders require IOC to be set. --- freqtrade/exchange/gateio.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index f69a0dc02..fd9a2b2b3 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -3,6 +3,7 @@ import logging from datetime import datetime from typing import Dict, List, Optional, Tuple +from freqtrade.constants import BuySell from freqtrade.enums import MarginMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange @@ -24,6 +25,8 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, "ohlcv_volume_currency": "quote", + "time_in_force_parameter": "timeInForce", + "order_time_in_force": ['gtc', 'ioc'], "stoploss_order_types": {"limit": "limit"}, "stoploss_on_exchange": True, } @@ -46,6 +49,27 @@ class Gateio(Exchange): raise OperationalException( f'Exchange {self.name} does not support market orders.') + def _get_params( + self, + side: BuySell, + ordertype: str, + leverage: float, + reduceOnly: bool, + time_in_force: str = 'gtc', + ) -> Dict: + params = super()._get_params( + side=side, + ordertype=ordertype, + leverage=leverage, + reduceOnly=reduceOnly, + time_in_force=time_in_force, + ) + if ordertype == 'market' and self.trading_mode == TradingMode.FUTURES: + params['type'] = 'market' + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: 'ioc'}) + return params + def get_trades_for_order(self, order_id: str, pair: str, since: datetime, params: Optional[Dict] = None) -> List: trades = super().get_trades_for_order(order_id, pair, since, params) @@ -60,7 +84,8 @@ class Gateio(Exchange): pair_fees = self._trading_fees.get(pair, {}) if pair_fees: for idx, trade in enumerate(trades): - if trade.get('fee', {}).get('cost') is None: + fee = trade.get('fee', {}) + if fee and fee.get('cost') is None: takerOrMaker = trade.get('takerOrMaker', 'taker') if pair_fees.get(takerOrMaker) is not None: trades[idx]['fee'] = { From 017fd03180e09549c0fc4604e19581bf54a66de1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:05:22 +0200 Subject: [PATCH 13/35] Fix but with late entries in backtesting --- freqtrade/data/history/history_utils.py | 2 +- freqtrade/optimize/backtesting.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index bead59814..c972c841c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -221,7 +221,7 @@ def _download_pair_history(pair: str, *, prepend=prepend) logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, ' - f'{candle_type} and store in {datadir}.' + f'{candle_type} and store in {datadir}. ' f'From {format_ms_time(since_ms) if since_ms else "start"} to ' f'{format_ms_time(until_ms) if until_ms else "now"}' ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 77eb12419..f7d92081f 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1078,6 +1078,8 @@ class Backtesting: open_trade_count += 1 # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.") open_trades[pair].append(trade) + LocalTrade.add_bt_trade(trade) + self.wallets.update() for trade in list(open_trades[pair]): # 3. Process entry orders. @@ -1085,7 +1087,6 @@ class Backtesting: if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time, trade) trade.open_order_id = None - LocalTrade.add_bt_trade(trade) self.wallets.update() # 4. Create exit orders (if any) From d62273294d4cb2ba23e94f733a08dfa02000dee3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:10:33 +0200 Subject: [PATCH 14/35] Update /help for /fx to align with actual command name closes #6985 --- docs/telegram-usage.md | 5 +++-- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 6e21d3689..773a1b67a 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -171,8 +171,8 @@ official commands. You can ask at any moment for help with `/help`. | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). | `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) -| `/forceexit ` | Instantly exits the given trade (Ignoring `minimum_roi`). -| `/forceexit all` | Instantly exits all open trades (Ignoring `minimum_roi`). +| `/forceexit | /fx ` | Instantly exits the given trade (Ignoring `minimum_roi`). +| `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`). | `/fx` | alias for `/forceexit` | `/forcelong [rate]` | Instantly buys the given pair. Rate is optional and only applies to limit orders. (`force_entry_enable` must be set to True) | `/forceshort [rate]` | Instantly shorts the given pair. Rate is optional and only applies to limit orders. This will only work on non-spot markets. (`force_entry_enable` must be set to True) @@ -281,6 +281,7 @@ Starting capital is either taken from the `available_capital` setting, or calcul !!! Tip You can get a list of all open trades by calling `/forceexit` without parameter, which will show a list of buttons to simply exit a trade. + This command has an alias in `/fx` - which has the same capabilities, but is faster to type in "emergency" situations. ### /forcelong [rate] | /forceshort [rate] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2e1d23621..7f5da8872 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1395,7 +1395,7 @@ class Telegram(RPCHandler): "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" - "*/fe |all:* `Alias to /forceexit`\n" + "*/fx |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" "*/whitelist:* `Show current whitelist` \n" From 03815cb81bde3f568f3319760c577b8229878eb0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:23:16 +0200 Subject: [PATCH 15/35] Use fstrings in telegram messaging --- freqtrade/rpc/telegram.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7f5da8872..c595018d4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -326,33 +326,33 @@ class Telegram(RPCHandler): elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' - message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling {message_side} Order for {pair} (#{trade_id}). " - "Reason: {reason}.".format(**msg)) + message = (f"\N{WARNING SIGN} *{msg['exchange']}:* " + f"Cancelling {msg['message_side']} Order for {msg['pair']} " + f"(#{msg['trade_id']}). Reason: {msg['reason']}.") elif msg_type == RPCMessageType.PROTECTION_TRIGGER: message = ( - "*Protection* triggered due to {reason}. " - "`{pair}` will be locked until `{lock_end_time}`." - ).format(**msg) + f"*Protection* triggered due to {msg['reason']}. " + f"`{msg['pair']}` will be locked until `{msg['lock_end_time']}`." + ) elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: message = ( - "*Protection* triggered due to {reason}. " - "*All pairs* will be locked until `{lock_end_time}`." - ).format(**msg) + f"*Protection* triggered due to {msg['reason']}. " + f"*All pairs* will be locked until `{msg['lock_end_time']}`." + ) elif msg_type == RPCMessageType.STATUS: - message = '*Status:* `{status}`'.format(**msg) + message = f"*Status:* `{msg['status']}`" elif msg_type == RPCMessageType.WARNING: - message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) + message = f"\N{WARNING SIGN} *Warning:* `{msg['status']}`" elif msg_type == RPCMessageType.STARTUP: - message = '{status}'.format(**msg) + message = f"{msg['status']}" else: - raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + raise NotImplementedError(f"Unknown message type: {msg_type}") return message def send_msg(self, msg: Dict[str, Any]) -> None: @@ -867,7 +867,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_start() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _stop(self, update: Update, context: CallbackContext) -> None: @@ -879,7 +879,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_stop() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _reload_config(self, update: Update, context: CallbackContext) -> None: @@ -891,7 +891,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_reload_config() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _stopbuy(self, update: Update, context: CallbackContext) -> None: @@ -903,7 +903,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_stopbuy() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _force_exit(self, update: Update, context: CallbackContext) -> None: @@ -1065,9 +1065,9 @@ class Telegram(RPCHandler): trade_id = int(context.args[0]) msg = self._rpc._rpc_delete(trade_id) self._send_msg(( - '`{result_msg}`\n' + f"`{msg['result_msg']}`\n" 'Please make sure to take care of this asset on the exchange manually.' - ).format(**msg)) + )) except RPCException as e: self._send_msg(str(e)) From d77ce468ea8fd196b09b36aafc98dcde8bebbfe5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:35:19 +0200 Subject: [PATCH 16/35] Add "dry" hint to buy/sell messages part of #6962 --- freqtrade/rpc/telegram.py | 14 +++++++++++--- tests/rpc/test_rpc_telegram.py | 19 +++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c595018d4..15e919e30 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -235,6 +235,14 @@ class Telegram(RPCHandler): # This can take up to `timeout` from the call to `start_polling`. self._updater.stop() + def _exchange_from_msg(self, msg: Dict[str, Any]) -> str: + """ + Extracts the exchange name from the given message. + :param msg: The message to extract the exchange name from. + :return: The exchange name. + """ + return f"{msg['exchange']}{' (dry)' if self._config['dry_run'] else ''}" + def _format_entry_msg(self, msg: Dict[str, Any]) -> str: if self._rpc._fiat_converter: msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( @@ -247,7 +255,7 @@ class Telegram(RPCHandler): entry_side = ({'enter': 'Long', 'entered': 'Longed'} if msg['direction'] == 'Long' else {'enter': 'Short', 'entered': 'Shorted'}) message = ( - f"{emoji} *{msg['exchange']}:*" + f"{emoji} *{self._exchange_from_msg(msg)}:*" f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}" f" (#{msg['trade_id']})\n" ) @@ -296,7 +304,7 @@ class Telegram(RPCHandler): msg['profit_extra'] = '' is_fill = msg['type'] == RPCMessageType.EXIT_FILL message = ( - f"{msg['emoji']} *{msg['exchange']}:* " + f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* " f"{'Exited' if is_fill else 'Exiting'} {msg['pair']} (#{msg['trade_id']})\n" f"*{'Profit' if is_fill else 'Unrealized Profit'}:* " f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n" @@ -326,7 +334,7 @@ class Telegram(RPCHandler): elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' - message = (f"\N{WARNING SIGN} *{msg['exchange']}:* " + message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* " f"Cancelling {msg['message_side']} Order for {msg['pair']} " f"(#{msg['trade_id']}). Reason: {msg['reason']}.") diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3bd817ac7..d6845be57 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1680,7 +1680,7 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog, message_type, leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{LARGE BLUE CIRCLE} *Binance:* {enter} ETH/BTC (#1)\n' + f'\N{LARGE BLUE CIRCLE} *Binance (dry):* {enter} ETH/BTC (#1)\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' f'{leverage_text}' @@ -1720,7 +1720,7 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker, message_type, en 'pair': 'ETH/BTC', 'reason': CANCEL_REASON['TIMEOUT'] }) - assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Binance:* ' + assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Binance (dry):* ' 'Cancelling enter Order for ETH/BTC (#1). ' 'Reason: cancelled due to timeout.') @@ -1782,7 +1782,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{CHECK MARK} *Binance:* {entered}ed ETH/BTC (#1)\n' + f'\N{CHECK MARK} *Binance (dry):* {entered}ed ETH/BTC (#1)\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' f"{leverage_text}" @@ -1820,7 +1820,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' @@ -1854,7 +1854,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' @@ -1883,10 +1883,12 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: 'reason': 'Cancelled on exchange' }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Cancelling exit Order for KEY/ETH (#1).' + '\N{WARNING SIGN} *Binance (dry):* Cancelling exit Order for KEY/ETH (#1).' ' Reason: Cancelled on exchange.') msg_mock.reset_mock() + # Test with live mode (no dry appendix) + telegram._config['dry_run'] = False telegram.send_msg({ 'type': RPCMessageType.EXIT_CANCEL, 'trade_id': 1, @@ -1935,7 +1937,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exited KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n' '*Profit:* `-57.41%`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' @@ -1991,6 +1993,7 @@ def test_send_msg_unknown_type(default_conf, mocker) -> None: def test_send_msg_buy_notification_no_fiat( default_conf, mocker, message_type, enter, enter_signal, leverage) -> None: del default_conf['fiat_display_currency'] + default_conf['dry_run'] = False telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram.send_msg({ @@ -2060,7 +2063,7 @@ def test_send_msg_sell_notification_no_fiat( leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' From 6a15d36d14525ff20083a788848230ee3150fa45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:14:28 +0200 Subject: [PATCH 17/35] Add Drawdown and profit_factor to /profit #6816 --- freqtrade/optimize/optimize_reports.py | 2 ++ freqtrade/rpc/api_server/api_schemas.py | 3 +++ freqtrade/rpc/rpc.py | 23 +++++++++++++++++++++++ freqtrade/rpc/telegram.py | 11 ++++++++--- tests/rpc/test_rpc_apiserver.py | 15 ++++++++++++--- tests/rpc/test_rpc_telegram.py | 5 +++-- 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 44b524a4c..79cb8a2bd 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -497,8 +497,10 @@ def generate_strategy_stats(pairlist: List[str], (drawdown_abs, drawdown_start, drawdown_end, high_val, low_val, max_drawdown) = calculate_max_drawdown( results, value_col='profit_abs', starting_balance=start_balance) + # max_relative_drawdown = Underwater (_, _, _, _, _, max_relative_drawdown) = calculate_max_drawdown( results, value_col='profit_abs', starting_balance=start_balance, relative=True) + strat_stats.update({ 'max_drawdown': max_drawdown_legacy, # Deprecated - do not use 'max_drawdown_account': max_drawdown, diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 11fdc0121..fda2d7ea0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -104,6 +104,9 @@ class Profit(BaseModel): best_pair_profit_ratio: float winning_trades: int losing_trades: int + profit_factor: float + max_drawdown: float + max_drawdown_abs: float class SellReason(BaseModel): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8b1cdb851..bae90b3bc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -18,6 +18,7 @@ from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.data.history import load_data +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State, TradingMode) from freqtrade.exceptions import ExchangeError, PricingError @@ -415,6 +416,8 @@ class RPC: durations = [] winning_trades = 0 losing_trades = 0 + winning_profit = 0.0 + losing_profit = 0.0 for trade in trades: current_rate: float = 0.0 @@ -430,8 +433,10 @@ class RPC: profit_closed_ratio.append(profit_ratio) if trade.close_profit >= 0: winning_trades += 1 + winning_profit += trade.close_profit_abs else: losing_trades += 1 + losing_profit += trade.close_profit_abs else: # Get current rate try: @@ -470,6 +475,21 @@ class RPC: profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance + profit_factor = winning_profit / abs(losing_profit) if losing_profit else float('inf') + + trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT), + 'profit_abs': trade.close_profit_abs} + for trade in trades if not trade.is_open]) + max_drawdown_abs = 0.0 + max_drawdown = 0.0 + if len(trades_df) > 0: + try: + (max_drawdown_abs, _, _, _, _, max_drawdown) = calculate_max_drawdown( + trades_df, value_col='profit_abs', starting_balance=starting_balance) + except ValueError: + # ValueError if no losing trade. + pass + profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, stake_currency, @@ -508,6 +528,9 @@ class RPC: 'best_pair_profit_ratio': best_pair[1] if best_pair else 0, 'winning_trades': winning_trades, 'losing_trades': losing_trades, + 'profit_factor': profit_factor, + 'max_drawdown': max_drawdown, + 'max_drawdown_abs': max_drawdown_abs, } def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 15e919e30..a7130d691 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -730,12 +730,17 @@ class Telegram(RPCHandler): f"*Total Trade Count:* `{trade_count}`\n" f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " f"`{first_trade_date}`\n" - f"*Latest Trade opened:* `{latest_trade_date}\n`" + f"*Latest Trade opened:* `{latest_trade_date}`\n" f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" ) if stats['closed_trade_count'] > 0: - markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" - f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`") + markdown_msg += ( + f"\n*Avg. Duration:* `{avg_duration}`\n" + f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n" + f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" + f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " + f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" + ) self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", query=update.callback_query) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index ada1a82ec..afbc92c5d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -724,7 +724,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': -83.19455985, 'profit_closed_ratio_mean': -0.0075, 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, - 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2} + 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2, + 'profit_factor': 0.0, + } ), ( False, @@ -737,7 +739,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': 9.124559849999999, 'profit_closed_ratio_mean': 0.0075, 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, - 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0} + 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0, + 'profit_factor': None, + } ), ( None, @@ -750,7 +754,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': -67.02260985, 'profit_closed_ratio_mean': 0.0025, 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, - 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1} + 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1, + 'profit_factor': 0.02775724835771106, + } ) ]) def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected): @@ -803,6 +809,9 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'closed_trade_count': 2, 'winning_trades': expected['winning_trades'], 'losing_trades': expected['losing_trades'], + 'profit_factor': expected['profit_factor'], + 'max_drawdown': ANY, + 'max_drawdown_abs': ANY, } diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index d6845be57..65917a6e2 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -704,11 +704,12 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '∙ `6.253 USD`' in msg_mock.call_args_list[-1][0][0] assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] + assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] + assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) -def test_telegram_stats(default_conf, update, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, mocker, is_short) -> None: +def test_telegram_stats(default_conf, update, ticker, fee, mocker, is_short) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', From 40c9abc7e1b2120d80884f00f260ce635a52d74a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:40:32 +0200 Subject: [PATCH 18/35] Add trading volume to /profit output --- freqtrade/persistence/trade_model.py | 16 ++++++++++++++++ freqtrade/rpc/rpc.py | 2 ++ freqtrade/rpc/telegram.py | 1 + tests/rpc/test_rpc_telegram.py | 1 + 4 files changed, 20 insertions(+) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index eb405942a..3a52c0660 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1352,3 +1352,19 @@ class Trade(_DECL_BASE, LocalTrade): .group_by(Trade.pair) \ .order_by(desc('profit_sum')).first() return best_pair + + @staticmethod + def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float: + """ + Get Trade volume based on Orders + NOTE: Not supported in Backtesting. + :returns: Tuple containing (pair, profit_sum) + """ + trading_volume = Order.query.with_entities( + func.sum(Order.cost).label('volume') + ).filter( + (Order.order_filled_date >= start_date) + & (Order.status == 'closed') + ) \ + .scalar() + return trading_volume diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index bae90b3bc..31fe4c469 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -452,6 +452,7 @@ class RPC: profit_all_ratio.append(profit_ratio) best_pair = Trade.get_best_pair(start_date) + trading_volume = Trade.get_trading_volume(start_date) # Prepare data to display profit_closed_coin_sum = round(sum(profit_closed_coin), 8) @@ -531,6 +532,7 @@ class RPC: 'profit_factor': profit_factor, 'max_drawdown': max_drawdown, 'max_drawdown_abs': max_drawdown_abs, + 'trading_volume': trading_volume, } def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a7130d691..58bfc6bf7 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -737,6 +737,7 @@ class Telegram(RPCHandler): markdown_msg += ( f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n" + f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n" f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 65917a6e2..e36d98083 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -706,6 +706,7 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] + assert '*Trading volume:* `60 USDT`' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) From b7e4dea6c5d6004e51a823f4bacdb5ac200d3016 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:43:50 +0200 Subject: [PATCH 19/35] Document new Profit metrics --- docs/telegram-usage.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 773a1b67a..95e7eaa16 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -270,10 +270,15 @@ Return a summary of your profit/loss and performance. > **Latest Trade opened:** `2 minutes ago` > **Avg. Duration:** `2:33:45` > **Best Performing:** `PAY/BTC: 50.23%` +> **Trading volume:** `0.5 BTC` +> **Profit factor:** `1.04` +> **Max Drawdown:** `9.23% (0.01255 BTC)` The relative profit of `1.2%` is the average profit per trade. The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. +Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. +Max drawdown corresponds to the backtesting metric `Absolute Drawdown (Account)` - calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`. ### /forceexit From 8c46d19071d96dae2eca4800e982b0e668e461d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:27:54 +0200 Subject: [PATCH 20/35] Fix backtesting bug balance was never released on cancelled trades --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f7d92081f..6eeefbfac 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1055,6 +1055,7 @@ class Backtesting: # Close trade open_trade_count -= 1 open_trades[pair].remove(t) + LocalTrade.trades_open.remove(t) self.wallets.update() # 2. Process entries. From 53bfa7931d1e4ac02a76a5b9be17d4fd6c55725a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:32:22 +0200 Subject: [PATCH 21/35] Add rudimentary test for prior bug Test fails without the fix in 8c46d19071d96dae2eca4800e982b0e668e461d6 --- tests/optimize/test_backtest_detail.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 4b4c446e0..a18196507 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -7,6 +7,7 @@ import pytest from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType from freqtrade.optimize.backtesting import Backtesting +from freqtrade.persistence.trade_model import LocalTrade from tests.conftest import patch_exchange from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset, tests_timeframe) @@ -964,5 +965,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) assert res.open_date == _get_frame_time_from_offset(trade.open_tick) assert res.close_date == _get_frame_time_from_offset(trade.close_tick) assert res.is_short == trade.is_short + assert len(LocalTrade.trades) == len(data.trades) + assert len(LocalTrade.trades_open) == 0 backtesting.cleanup() del backtesting From 474e6705e622aa7372c7a9b203df8553dbdcded3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:27:43 +0200 Subject: [PATCH 22/35] Add Profit factor to backtesting --- docs/backtesting.md | 4 ++++ docs/telegram-usage.md | 6 +++--- freqtrade/optimize/optimize_reports.py | 6 ++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 76718d206..50fc96923 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -300,6 +300,7 @@ A backtesting result will look like that: | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | | CAGR % | 460.87% | +| Profit factor | 1.11 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -399,6 +400,7 @@ It contains some useful key metrics about performance of your strategy on backte | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | | CAGR % | 460.87% | +| Profit factor | 1.11 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -444,6 +446,8 @@ It contains some useful key metrics about performance of your strategy on backte - `Final balance`: Final balance - starting balance + absolute profit. - `Absolute profit`: Profit made in stake currency. - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. +- `CAGR %`: Compound annual growth rate. +- `Profit factor`: profit / loss. - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. - `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 95e7eaa16..2145797b4 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -275,9 +275,9 @@ Return a summary of your profit/loss and performance. > **Max Drawdown:** `9.23% (0.01255 BTC)` The relative profit of `1.2%` is the average profit per trade. -The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. -Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. -Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. +The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. +Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. +Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. Max drawdown corresponds to the backtesting metric `Absolute Drawdown (Account)` - calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`. ### /forceexit diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 79cb8a2bd..44ac4a5b3 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -416,6 +416,9 @@ def generate_strategy_stats(pairlist: List[str], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + winning_profit = results.loc[results['profit_abs'] > 0, 'profit_abs'].sum() + losing_profit = results.loc[results['profit_abs'] < 0, 'profit_abs'].sum() + profit_factor = winning_profit / abs(losing_profit) if losing_profit else 0.0 backtest_days = (max_date - min_date).days or 1 strat_stats = { @@ -443,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str], 'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(), 'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(), 'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']), + 'profit_factor': profit_factor, 'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT), 'backtest_start_ts': int(min_date.timestamp() * 1000), 'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT), @@ -779,6 +783,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: strat_results['stake_currency'])), ('Total profit %', f"{strat_results['profit_total']:.2%}"), ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), + ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor' + in strat_results else 'N/A'), ('Trades per day', strat_results['trades_per_day']), ('Avg. daily profit %', f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), From 0168343b7656eb572c0bda50daa774e2f9549e5f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:53:25 +0200 Subject: [PATCH 23/35] Add trading-volume to api schema --- freqtrade/persistence/trade_model.py | 7 +++---- freqtrade/rpc/api_server/api_schemas.py | 1 + tests/conftest_trades.py | 14 ++++++++++++++ tests/rpc/test_rpc_apiserver.py | 7 ++++--- tests/test_persistence.py | 1 + 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3a52c0660..39ebd75b4 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1363,8 +1363,7 @@ class Trade(_DECL_BASE, LocalTrade): trading_volume = Order.query.with_entities( func.sum(Order.cost).label('volume') ).filter( - (Order.order_filled_date >= start_date) - & (Order.status == 'closed') - ) \ - .scalar() + Order.order_filled_date >= start_date, + Order.status == 'closed' + ).scalar() return trading_volume diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index fda2d7ea0..7566e2ac0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -107,6 +107,7 @@ class Profit(BaseModel): profit_factor: float max_drawdown: float max_drawdown_abs: float + trading_volume: Optional[float] class SellReason(BaseModel): diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 006eab98f..1a8cf3183 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -29,6 +29,7 @@ def mock_order_1(is_short: bool): 'average': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -65,6 +66,7 @@ def mock_order_2(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -79,6 +81,7 @@ def mock_order_2_sell(is_short: bool): 'price': 0.128, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -126,6 +129,7 @@ def mock_order_3(is_short: bool): 'price': 0.05, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -141,6 +145,7 @@ def mock_order_3_sell(is_short: bool): 'average': 0.06, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -186,6 +191,7 @@ def mock_order_4(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 0.0, + 'cost': 15.129, 'remaining': 123.0, } @@ -225,6 +231,7 @@ def mock_order_5(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -239,6 +246,7 @@ def mock_order_5_stoploss(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 0.0, + 'cost': 0.0, 'remaining': 123.0, } @@ -281,6 +289,7 @@ def mock_order_6(is_short: bool): 'price': 0.15, 'amount': 2.0, 'filled': 2.0, + 'cost': 0.3, 'remaining': 0.0, } @@ -295,6 +304,7 @@ def mock_order_6_sell(is_short: bool): 'price': 0.15 if is_short else 0.20, 'amount': 2.0, 'filled': 0.0, + 'cost': 0.0, 'remaining': 2.0, } @@ -337,6 +347,7 @@ def short_order(): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -351,6 +362,7 @@ def exit_short_order(): 'price': 0.128, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.744, 'remaining': 0.0, } @@ -424,6 +436,7 @@ def leverage_order(): 'amount': 123.0, 'filled': 123.0, 'remaining': 0.0, + 'cost': 15.129, 'leverage': 5.0 } @@ -439,6 +452,7 @@ def leverage_order_sell(): 'amount': 123.0, 'filled': 123.0, 'remaining': 0.0, + 'cost': 15.744, 'leverage': 5.0 } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index afbc92c5d..b0ff5e1b2 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -725,7 +725,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2, - 'profit_factor': 0.0, + 'profit_factor': 0.0, 'trading_volume': 91.074, } ), ( @@ -740,7 +740,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0, - 'profit_factor': None, + 'profit_factor': None, 'trading_volume': 91.074, } ), ( @@ -755,7 +755,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1, - 'profit_factor': 0.02775724835771106, + 'profit_factor': 0.02775724835771106, 'trading_volume': 91.074, } ) ]) @@ -812,6 +812,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'profit_factor': expected['profit_factor'], 'max_drawdown': ANY, 'max_drawdown_abs': ANY, + 'trading_volume': expected['trading_volume'], } diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8c12d2ea0..357233dfa 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -2269,6 +2269,7 @@ def test_Trade_object_idem(): 'get_exit_reason_performance', 'get_enter_tag_performance', 'get_mix_tag_performance', + 'get_trading_volume', ) From 0809f9aef69776805912bf8b32621a0a3d481959 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 17:44:15 +0200 Subject: [PATCH 24/35] Add offset to trade response --- freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/rpc.py | 1 + tests/rpc/test_rpc_apiserver.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 7566e2ac0..333f2fe6e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -283,6 +283,7 @@ class OpenTradeSchema(TradeSchema): class TradeResponse(BaseModel): trades: List[TradeSchema] trades_count: int + offset: int total_trades: int diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 31fe4c469..dbbb78c98 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -365,6 +365,7 @@ class RPC: return { "trades": output, "trades_count": len(output), + "offset": offset, "total_trades": Trade.get_trades([Trade.is_open.is_(False)]).count(), } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b0ff5e1b2..c0de54c6d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -578,9 +578,10 @@ def test_api_trades(botclient, mocker, fee, markets, is_short): ) rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) - assert len(rc.json()) == 3 + assert len(rc.json()) == 4 assert rc.json()['trades_count'] == 0 assert rc.json()['total_trades'] == 0 + assert rc.json()['offset'] == 0 create_mock_trades(fee, is_short=is_short) Trade.query.session.flush() From 0d967f93baf69e08bda264df0702f4a433abf64b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jun 2022 16:13:00 +0200 Subject: [PATCH 25/35] Improve performance of some RPC calls These don't need orders to be loaded. As a side-effect, this will also reduce the strain on the database. --- freqtrade/persistence/trade_model.py | 13 +++++++++---- freqtrade/rpc/rpc.py | 5 +++-- tests/test_persistence.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 39ebd75b4..0c8c985c8 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func) -from sqlalchemy.orm import Query, relationship +from sqlalchemy.orm import Query, lazyload, relationship from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort from freqtrade.enums import ExitType, TradingMode @@ -1115,7 +1115,7 @@ class Trade(_DECL_BASE, LocalTrade): ) @staticmethod - def get_trades(trade_filter=None) -> Query: + def get_trades(trade_filter=None, include_orders: bool = True) -> Query: """ Helper function to query Trades using filters. NOTE: Not supported in Backtesting. @@ -1130,9 +1130,14 @@ class Trade(_DECL_BASE, LocalTrade): if trade_filter is not None: if not isinstance(trade_filter, list): trade_filter = [trade_filter] - return Trade.query.filter(*trade_filter) + this_query = Trade.query.filter(*trade_filter) else: - return Trade.query + this_query = Trade.query + if not include_orders: + # Don't load order relations + # Consider using noload or raiseload instead of lazyload + this_query = this_query.options(lazyload(Trade.orders)) + return this_query @staticmethod def get_open_order_trades() -> List['Trade']: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dbbb78c98..c42a6f683 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -380,7 +380,7 @@ class RPC: return 'losses' else: return 'draws' - trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)]) + trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False) # Sell reason exit_reasons = {} for trade in trades: @@ -408,7 +408,8 @@ class RPC: """ Returns cumulative profit statistics """ trade_filter = ((Trade.is_open.is_(False) & (Trade.close_date >= start_date)) | Trade.is_open.is_(True)) - trades: List[Trade] = Trade.get_trades(trade_filter).order_by(Trade.id).all() + trades: List[Trade] = Trade.get_trades( + trade_filter, include_orders=False).order_by(Trade.id).all() profit_all_coin = [] profit_all_ratio = [] diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 357233dfa..deaad258b 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -2075,6 +2075,24 @@ def test_get_trades_proxy(fee, use_db, is_short): Trade.use_db = True +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize('is_short', [True, False]) +def test_get_trades__query(fee, is_short): + query = Trade.get_trades([]) + # without orders there should be no join issued. + query1 = Trade.get_trades([], include_orders=False) + + assert "JOIN orders" in str(query) + assert "JOIN orders" not in str(query1) + + create_mock_trades(fee, is_short) + query = Trade.get_trades([]) + query1 = Trade.get_trades([], include_orders=False) + + assert "JOIN orders" in str(query) + assert "JOIN orders" not in str(query1) + + def test_get_trades_backtest(): Trade.use_db = False with pytest.raises(NotImplementedError, match=r"`Trade.get_trades\(\)` not .*"): From 8406010260e3d79be0638b375b63fd447d4711da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 03:01:26 +0000 Subject: [PATCH 26/35] Bump types-cachetools from 5.0.1 to 5.0.2 Bumps [types-cachetools](https://github.com/python/typeshed) from 5.0.1 to 5.0.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 19912d59c..53c85f176 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ time-machine==2.7.0 nbconvert==6.5.0 # mypy types -types-cachetools==5.0.1 +types-cachetools==5.0.2 types-filelock==3.2.7 types-requests==2.27.30 types-tabulate==0.8.9 From 55fb7656dfdafebf754d6a5b14c8bdd0eca2eb29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 06:58:41 +0200 Subject: [PATCH 27/35] Update pre-commit cachetools --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5c1a36f5..e057627cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: mypy exclude: build_helpers additional_dependencies: - - types-cachetools==5.0.1 + - types-cachetools==5.0.2 - types-filelock==3.2.7 - types-requests==2.27.30 - types-tabulate==0.8.9 From 0804fc7a3af91c25b291bd991132cf77d8d42946 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:01:35 +0200 Subject: [PATCH 28/35] CI should run ccxt tests only once --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 551268af7..2f67ec5fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: - name: Tests incl. ccxt compatibility tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun - if: matrix.python-version == '3.9' + if: matrix.python-version == '3.9' and matrix.os == 'ubuntu-22.04' - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.9') From f9668ede4a54edda390197b3261504b3d526c77d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:02:12 +0200 Subject: [PATCH 29/35] Fix CI Syntax error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f67ec5fe..81a18c4c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: - name: Tests incl. ccxt compatibility tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun - if: matrix.python-version == '3.9' and matrix.os == 'ubuntu-22.04' + if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-22.04' - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.9') From 50c19ece53644808adf80a95eebefdf6fe3f4c6d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:05:51 +0200 Subject: [PATCH 30/35] Fix ccxt test gateio flukyness --- tests/exchange/test_ccxt_compat.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index e016873cb..50154bcaf 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -199,8 +199,13 @@ class TestCCXTExchange(): l2 = exchange.fetch_l2_order_book(pair) assert 'asks' in l2 assert 'bids' in l2 + assert len(l2['asks']) >= 1 + assert len(l2['bids']) >= 1 l2_limit_range = exchange._ft_has['l2_limit_range'] l2_limit_range_required = exchange._ft_has['l2_limit_range_required'] + if exchangename == 'gateio': + # TODO: Gateio is unstable here at the moment, ignoring the limit partially. + return for val in [1, 2, 5, 25, 100]: l2 = exchange.fetch_l2_order_book(pair, val) if not l2_limit_range or val in l2_limit_range: From 996372b8f6c18b9721034bd6abfd41532f3e2b62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 05:06:39 +0000 Subject: [PATCH 31/35] Bump colorama from 0.4.4 to 0.4.5 Bumps [colorama](https://github.com/tartley/colorama) from 0.4.4 to 0.4.5. - [Release notes](https://github.com/tartley/colorama/releases) - [Changelog](https://github.com/tartley/colorama/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tartley/colorama/compare/0.4.4...0.4.5) --- updated-dependencies: - dependency-name: colorama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b2dbd921e..bd28e3768 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ aiofiles==0.8.0 psutil==5.9.1 # Support for colorized terminal output -colorama==0.4.4 +colorama==0.4.5 # Building config files interactively questionary==1.10.0 prompt-toolkit==3.0.29 From e1e3a903f98aebd0d2db6b9c6e0e5f51b70075c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 05:07:35 +0000 Subject: [PATCH 32/35] Bump ccxt from 1.87.12 to 1.88.15 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.87.12 to 1.88.15. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.87.12...1.88.15) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b2dbd921e..ec244806d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.87.12 +ccxt==1.88.15 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 1cd2b0504a9e1c684326ab59a6dcb6f2a8eb85a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:15:15 +0200 Subject: [PATCH 33/35] Run regular tests for 3.9 under other ubuntu systems --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81a18c4c9..818d250ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc - if: matrix.python-version != '3.9' + if: matrix.python-version != '3.9' && matrix.os != 'ubuntu-22.04' - name: Tests incl. ccxt compatibility tests run: | From 3189b284c014c4624ba27fc8abbde982c5b36c0c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 08:04:34 +0200 Subject: [PATCH 34/35] Fix tests condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 818d250ca..4fe1ad853 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc - if: matrix.python-version != '3.9' && matrix.os != 'ubuntu-22.04' + if: matrix.python-version != '3.9' || matrix.os != 'ubuntu-22.04' - name: Tests incl. ccxt compatibility tests run: | From 15fac746a8129e00acb86246e16b00fb7daea89f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 06:59:58 +0000 Subject: [PATCH 35/35] Bump mkdocs-material from 8.3.4 to 8.3.6 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.3.4 to 8.3.6. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.3.4...8.3.6) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 1f342ca02..6477ad23f 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.3.4 +mkdocs-material==8.3.6 mdx_truly_sane_lists==1.2 pymdown-extensions==9.5 jinja2==3.1.2