From fda8248d41bce429415cb20292550df6d7e5e654 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 22:43:24 +0200 Subject: [PATCH 01/25] Gateio allow market orders on futures markets --- freqtrade/exchange/gateio.py | 1 - tests/exchange/test_gateio.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index 4147e8290..f69a0dc02 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -40,7 +40,6 @@ class Gateio(Exchange): ] def validate_ordertypes(self, order_types: Dict) -> None: - super().validate_ordertypes(order_types) if self.trading_mode != TradingMode.FUTURES: if any(v == 'market' for k, v in order_types.items()): diff --git a/tests/exchange/test_gateio.py b/tests/exchange/test_gateio.py index 92f8186a6..cbd4776fb 100644 --- a/tests/exchange/test_gateio.py +++ b/tests/exchange/test_gateio.py @@ -33,6 +33,12 @@ def test_validate_order_types_gateio(default_conf, mocker): match=r'Exchange .* does not support market orders.'): ExchangeResolver.load_exchange('gateio', default_conf, True) + # market-orders supported on futures markets. + default_conf['trading_mode'] = 'futures' + default_conf['margin_mode'] = 'isolated' + ex = ExchangeResolver.load_exchange('gateio', default_conf, True) + assert ex + @pytest.mark.usefixtures("init_persistence") def test_fetch_stoploss_order_gateio(default_conf, mocker): From 616bf315cbf8647f5f9114aee74e648f747b9a27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 23:02:39 +0200 Subject: [PATCH 02/25] gateio: futures market orders require IOC to be set. --- freqtrade/exchange/gateio.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index f69a0dc02..fd9a2b2b3 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -3,6 +3,7 @@ import logging from datetime import datetime from typing import Dict, List, Optional, Tuple +from freqtrade.constants import BuySell from freqtrade.enums import MarginMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange @@ -24,6 +25,8 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, "ohlcv_volume_currency": "quote", + "time_in_force_parameter": "timeInForce", + "order_time_in_force": ['gtc', 'ioc'], "stoploss_order_types": {"limit": "limit"}, "stoploss_on_exchange": True, } @@ -46,6 +49,27 @@ class Gateio(Exchange): raise OperationalException( f'Exchange {self.name} does not support market orders.') + def _get_params( + self, + side: BuySell, + ordertype: str, + leverage: float, + reduceOnly: bool, + time_in_force: str = 'gtc', + ) -> Dict: + params = super()._get_params( + side=side, + ordertype=ordertype, + leverage=leverage, + reduceOnly=reduceOnly, + time_in_force=time_in_force, + ) + if ordertype == 'market' and self.trading_mode == TradingMode.FUTURES: + params['type'] = 'market' + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: 'ioc'}) + return params + def get_trades_for_order(self, order_id: str, pair: str, since: datetime, params: Optional[Dict] = None) -> List: trades = super().get_trades_for_order(order_id, pair, since, params) @@ -60,7 +84,8 @@ class Gateio(Exchange): pair_fees = self._trading_fees.get(pair, {}) if pair_fees: for idx, trade in enumerate(trades): - if trade.get('fee', {}).get('cost') is None: + fee = trade.get('fee', {}) + if fee and fee.get('cost') is None: takerOrMaker = trade.get('takerOrMaker', 'taker') if pair_fees.get(takerOrMaker) is not None: trades[idx]['fee'] = { From 017fd03180e09549c0fc4604e19581bf54a66de1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:05:22 +0200 Subject: [PATCH 03/25] Fix but with late entries in backtesting --- freqtrade/data/history/history_utils.py | 2 +- freqtrade/optimize/backtesting.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index bead59814..c972c841c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -221,7 +221,7 @@ def _download_pair_history(pair: str, *, prepend=prepend) logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, ' - f'{candle_type} and store in {datadir}.' + f'{candle_type} and store in {datadir}. ' f'From {format_ms_time(since_ms) if since_ms else "start"} to ' f'{format_ms_time(until_ms) if until_ms else "now"}' ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 77eb12419..f7d92081f 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1078,6 +1078,8 @@ class Backtesting: open_trade_count += 1 # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.") open_trades[pair].append(trade) + LocalTrade.add_bt_trade(trade) + self.wallets.update() for trade in list(open_trades[pair]): # 3. Process entry orders. @@ -1085,7 +1087,6 @@ class Backtesting: if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time, trade) trade.open_order_id = None - LocalTrade.add_bt_trade(trade) self.wallets.update() # 4. Create exit orders (if any) From d62273294d4cb2ba23e94f733a08dfa02000dee3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:10:33 +0200 Subject: [PATCH 04/25] Update /help for /fx to align with actual command name closes #6985 --- docs/telegram-usage.md | 5 +++-- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 6e21d3689..773a1b67a 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -171,8 +171,8 @@ official commands. You can ask at any moment for help with `/help`. | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). | `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) -| `/forceexit ` | Instantly exits the given trade (Ignoring `minimum_roi`). -| `/forceexit all` | Instantly exits all open trades (Ignoring `minimum_roi`). +| `/forceexit | /fx ` | Instantly exits the given trade (Ignoring `minimum_roi`). +| `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`). | `/fx` | alias for `/forceexit` | `/forcelong [rate]` | Instantly buys the given pair. Rate is optional and only applies to limit orders. (`force_entry_enable` must be set to True) | `/forceshort [rate]` | Instantly shorts the given pair. Rate is optional and only applies to limit orders. This will only work on non-spot markets. (`force_entry_enable` must be set to True) @@ -281,6 +281,7 @@ Starting capital is either taken from the `available_capital` setting, or calcul !!! Tip You can get a list of all open trades by calling `/forceexit` without parameter, which will show a list of buttons to simply exit a trade. + This command has an alias in `/fx` - which has the same capabilities, but is faster to type in "emergency" situations. ### /forcelong [rate] | /forceshort [rate] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2e1d23621..7f5da8872 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1395,7 +1395,7 @@ class Telegram(RPCHandler): "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" - "*/fe |all:* `Alias to /forceexit`\n" + "*/fx |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" "*/whitelist:* `Show current whitelist` \n" From 03815cb81bde3f568f3319760c577b8229878eb0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:23:16 +0200 Subject: [PATCH 05/25] Use fstrings in telegram messaging --- freqtrade/rpc/telegram.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7f5da8872..c595018d4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -326,33 +326,33 @@ class Telegram(RPCHandler): elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' - message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling {message_side} Order for {pair} (#{trade_id}). " - "Reason: {reason}.".format(**msg)) + message = (f"\N{WARNING SIGN} *{msg['exchange']}:* " + f"Cancelling {msg['message_side']} Order for {msg['pair']} " + f"(#{msg['trade_id']}). Reason: {msg['reason']}.") elif msg_type == RPCMessageType.PROTECTION_TRIGGER: message = ( - "*Protection* triggered due to {reason}. " - "`{pair}` will be locked until `{lock_end_time}`." - ).format(**msg) + f"*Protection* triggered due to {msg['reason']}. " + f"`{msg['pair']}` will be locked until `{msg['lock_end_time']}`." + ) elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: message = ( - "*Protection* triggered due to {reason}. " - "*All pairs* will be locked until `{lock_end_time}`." - ).format(**msg) + f"*Protection* triggered due to {msg['reason']}. " + f"*All pairs* will be locked until `{msg['lock_end_time']}`." + ) elif msg_type == RPCMessageType.STATUS: - message = '*Status:* `{status}`'.format(**msg) + message = f"*Status:* `{msg['status']}`" elif msg_type == RPCMessageType.WARNING: - message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) + message = f"\N{WARNING SIGN} *Warning:* `{msg['status']}`" elif msg_type == RPCMessageType.STARTUP: - message = '{status}'.format(**msg) + message = f"{msg['status']}" else: - raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + raise NotImplementedError(f"Unknown message type: {msg_type}") return message def send_msg(self, msg: Dict[str, Any]) -> None: @@ -867,7 +867,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_start() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _stop(self, update: Update, context: CallbackContext) -> None: @@ -879,7 +879,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_stop() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _reload_config(self, update: Update, context: CallbackContext) -> None: @@ -891,7 +891,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_reload_config() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _stopbuy(self, update: Update, context: CallbackContext) -> None: @@ -903,7 +903,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_stopbuy() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _force_exit(self, update: Update, context: CallbackContext) -> None: @@ -1065,9 +1065,9 @@ class Telegram(RPCHandler): trade_id = int(context.args[0]) msg = self._rpc._rpc_delete(trade_id) self._send_msg(( - '`{result_msg}`\n' + f"`{msg['result_msg']}`\n" 'Please make sure to take care of this asset on the exchange manually.' - ).format(**msg)) + )) except RPCException as e: self._send_msg(str(e)) From d77ce468ea8fd196b09b36aafc98dcde8bebbfe5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:35:19 +0200 Subject: [PATCH 06/25] Add "dry" hint to buy/sell messages part of #6962 --- freqtrade/rpc/telegram.py | 14 +++++++++++--- tests/rpc/test_rpc_telegram.py | 19 +++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c595018d4..15e919e30 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -235,6 +235,14 @@ class Telegram(RPCHandler): # This can take up to `timeout` from the call to `start_polling`. self._updater.stop() + def _exchange_from_msg(self, msg: Dict[str, Any]) -> str: + """ + Extracts the exchange name from the given message. + :param msg: The message to extract the exchange name from. + :return: The exchange name. + """ + return f"{msg['exchange']}{' (dry)' if self._config['dry_run'] else ''}" + def _format_entry_msg(self, msg: Dict[str, Any]) -> str: if self._rpc._fiat_converter: msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( @@ -247,7 +255,7 @@ class Telegram(RPCHandler): entry_side = ({'enter': 'Long', 'entered': 'Longed'} if msg['direction'] == 'Long' else {'enter': 'Short', 'entered': 'Shorted'}) message = ( - f"{emoji} *{msg['exchange']}:*" + f"{emoji} *{self._exchange_from_msg(msg)}:*" f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}" f" (#{msg['trade_id']})\n" ) @@ -296,7 +304,7 @@ class Telegram(RPCHandler): msg['profit_extra'] = '' is_fill = msg['type'] == RPCMessageType.EXIT_FILL message = ( - f"{msg['emoji']} *{msg['exchange']}:* " + f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* " f"{'Exited' if is_fill else 'Exiting'} {msg['pair']} (#{msg['trade_id']})\n" f"*{'Profit' if is_fill else 'Unrealized Profit'}:* " f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n" @@ -326,7 +334,7 @@ class Telegram(RPCHandler): elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' - message = (f"\N{WARNING SIGN} *{msg['exchange']}:* " + message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* " f"Cancelling {msg['message_side']} Order for {msg['pair']} " f"(#{msg['trade_id']}). Reason: {msg['reason']}.") diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3bd817ac7..d6845be57 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1680,7 +1680,7 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog, message_type, leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{LARGE BLUE CIRCLE} *Binance:* {enter} ETH/BTC (#1)\n' + f'\N{LARGE BLUE CIRCLE} *Binance (dry):* {enter} ETH/BTC (#1)\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' f'{leverage_text}' @@ -1720,7 +1720,7 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker, message_type, en 'pair': 'ETH/BTC', 'reason': CANCEL_REASON['TIMEOUT'] }) - assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Binance:* ' + assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Binance (dry):* ' 'Cancelling enter Order for ETH/BTC (#1). ' 'Reason: cancelled due to timeout.') @@ -1782,7 +1782,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{CHECK MARK} *Binance:* {entered}ed ETH/BTC (#1)\n' + f'\N{CHECK MARK} *Binance (dry):* {entered}ed ETH/BTC (#1)\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' f"{leverage_text}" @@ -1820,7 +1820,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' @@ -1854,7 +1854,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' @@ -1883,10 +1883,12 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: 'reason': 'Cancelled on exchange' }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Cancelling exit Order for KEY/ETH (#1).' + '\N{WARNING SIGN} *Binance (dry):* Cancelling exit Order for KEY/ETH (#1).' ' Reason: Cancelled on exchange.') msg_mock.reset_mock() + # Test with live mode (no dry appendix) + telegram._config['dry_run'] = False telegram.send_msg({ 'type': RPCMessageType.EXIT_CANCEL, 'trade_id': 1, @@ -1935,7 +1937,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exited KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n' '*Profit:* `-57.41%`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' @@ -1991,6 +1993,7 @@ def test_send_msg_unknown_type(default_conf, mocker) -> None: def test_send_msg_buy_notification_no_fiat( default_conf, mocker, message_type, enter, enter_signal, leverage) -> None: del default_conf['fiat_display_currency'] + default_conf['dry_run'] = False telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram.send_msg({ @@ -2060,7 +2063,7 @@ def test_send_msg_sell_notification_no_fiat( leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' From 6a15d36d14525ff20083a788848230ee3150fa45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:14:28 +0200 Subject: [PATCH 07/25] Add Drawdown and profit_factor to /profit #6816 --- freqtrade/optimize/optimize_reports.py | 2 ++ freqtrade/rpc/api_server/api_schemas.py | 3 +++ freqtrade/rpc/rpc.py | 23 +++++++++++++++++++++++ freqtrade/rpc/telegram.py | 11 ++++++++--- tests/rpc/test_rpc_apiserver.py | 15 ++++++++++++--- tests/rpc/test_rpc_telegram.py | 5 +++-- 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 44b524a4c..79cb8a2bd 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -497,8 +497,10 @@ def generate_strategy_stats(pairlist: List[str], (drawdown_abs, drawdown_start, drawdown_end, high_val, low_val, max_drawdown) = calculate_max_drawdown( results, value_col='profit_abs', starting_balance=start_balance) + # max_relative_drawdown = Underwater (_, _, _, _, _, max_relative_drawdown) = calculate_max_drawdown( results, value_col='profit_abs', starting_balance=start_balance, relative=True) + strat_stats.update({ 'max_drawdown': max_drawdown_legacy, # Deprecated - do not use 'max_drawdown_account': max_drawdown, diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 11fdc0121..fda2d7ea0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -104,6 +104,9 @@ class Profit(BaseModel): best_pair_profit_ratio: float winning_trades: int losing_trades: int + profit_factor: float + max_drawdown: float + max_drawdown_abs: float class SellReason(BaseModel): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8b1cdb851..bae90b3bc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -18,6 +18,7 @@ from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.data.history import load_data +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State, TradingMode) from freqtrade.exceptions import ExchangeError, PricingError @@ -415,6 +416,8 @@ class RPC: durations = [] winning_trades = 0 losing_trades = 0 + winning_profit = 0.0 + losing_profit = 0.0 for trade in trades: current_rate: float = 0.0 @@ -430,8 +433,10 @@ class RPC: profit_closed_ratio.append(profit_ratio) if trade.close_profit >= 0: winning_trades += 1 + winning_profit += trade.close_profit_abs else: losing_trades += 1 + losing_profit += trade.close_profit_abs else: # Get current rate try: @@ -470,6 +475,21 @@ class RPC: profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance + profit_factor = winning_profit / abs(losing_profit) if losing_profit else float('inf') + + trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT), + 'profit_abs': trade.close_profit_abs} + for trade in trades if not trade.is_open]) + max_drawdown_abs = 0.0 + max_drawdown = 0.0 + if len(trades_df) > 0: + try: + (max_drawdown_abs, _, _, _, _, max_drawdown) = calculate_max_drawdown( + trades_df, value_col='profit_abs', starting_balance=starting_balance) + except ValueError: + # ValueError if no losing trade. + pass + profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, stake_currency, @@ -508,6 +528,9 @@ class RPC: 'best_pair_profit_ratio': best_pair[1] if best_pair else 0, 'winning_trades': winning_trades, 'losing_trades': losing_trades, + 'profit_factor': profit_factor, + 'max_drawdown': max_drawdown, + 'max_drawdown_abs': max_drawdown_abs, } def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 15e919e30..a7130d691 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -730,12 +730,17 @@ class Telegram(RPCHandler): f"*Total Trade Count:* `{trade_count}`\n" f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " f"`{first_trade_date}`\n" - f"*Latest Trade opened:* `{latest_trade_date}\n`" + f"*Latest Trade opened:* `{latest_trade_date}`\n" f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" ) if stats['closed_trade_count'] > 0: - markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" - f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`") + markdown_msg += ( + f"\n*Avg. Duration:* `{avg_duration}`\n" + f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n" + f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" + f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " + f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" + ) self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", query=update.callback_query) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index ada1a82ec..afbc92c5d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -724,7 +724,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': -83.19455985, 'profit_closed_ratio_mean': -0.0075, 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, - 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2} + 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2, + 'profit_factor': 0.0, + } ), ( False, @@ -737,7 +739,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': 9.124559849999999, 'profit_closed_ratio_mean': 0.0075, 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, - 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0} + 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0, + 'profit_factor': None, + } ), ( None, @@ -750,7 +754,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': -67.02260985, 'profit_closed_ratio_mean': 0.0025, 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, - 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1} + 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1, + 'profit_factor': 0.02775724835771106, + } ) ]) def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected): @@ -803,6 +809,9 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'closed_trade_count': 2, 'winning_trades': expected['winning_trades'], 'losing_trades': expected['losing_trades'], + 'profit_factor': expected['profit_factor'], + 'max_drawdown': ANY, + 'max_drawdown_abs': ANY, } diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index d6845be57..65917a6e2 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -704,11 +704,12 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '∙ `6.253 USD`' in msg_mock.call_args_list[-1][0][0] assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] + assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] + assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) -def test_telegram_stats(default_conf, update, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, mocker, is_short) -> None: +def test_telegram_stats(default_conf, update, ticker, fee, mocker, is_short) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', From 40c9abc7e1b2120d80884f00f260ce635a52d74a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:40:32 +0200 Subject: [PATCH 08/25] Add trading volume to /profit output --- freqtrade/persistence/trade_model.py | 16 ++++++++++++++++ freqtrade/rpc/rpc.py | 2 ++ freqtrade/rpc/telegram.py | 1 + tests/rpc/test_rpc_telegram.py | 1 + 4 files changed, 20 insertions(+) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index eb405942a..3a52c0660 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1352,3 +1352,19 @@ class Trade(_DECL_BASE, LocalTrade): .group_by(Trade.pair) \ .order_by(desc('profit_sum')).first() return best_pair + + @staticmethod + def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float: + """ + Get Trade volume based on Orders + NOTE: Not supported in Backtesting. + :returns: Tuple containing (pair, profit_sum) + """ + trading_volume = Order.query.with_entities( + func.sum(Order.cost).label('volume') + ).filter( + (Order.order_filled_date >= start_date) + & (Order.status == 'closed') + ) \ + .scalar() + return trading_volume diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index bae90b3bc..31fe4c469 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -452,6 +452,7 @@ class RPC: profit_all_ratio.append(profit_ratio) best_pair = Trade.get_best_pair(start_date) + trading_volume = Trade.get_trading_volume(start_date) # Prepare data to display profit_closed_coin_sum = round(sum(profit_closed_coin), 8) @@ -531,6 +532,7 @@ class RPC: 'profit_factor': profit_factor, 'max_drawdown': max_drawdown, 'max_drawdown_abs': max_drawdown_abs, + 'trading_volume': trading_volume, } def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a7130d691..58bfc6bf7 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -737,6 +737,7 @@ class Telegram(RPCHandler): markdown_msg += ( f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n" + f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n" f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 65917a6e2..e36d98083 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -706,6 +706,7 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] + assert '*Trading volume:* `60 USDT`' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) From b7e4dea6c5d6004e51a823f4bacdb5ac200d3016 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:43:50 +0200 Subject: [PATCH 09/25] Document new Profit metrics --- docs/telegram-usage.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 773a1b67a..95e7eaa16 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -270,10 +270,15 @@ Return a summary of your profit/loss and performance. > **Latest Trade opened:** `2 minutes ago` > **Avg. Duration:** `2:33:45` > **Best Performing:** `PAY/BTC: 50.23%` +> **Trading volume:** `0.5 BTC` +> **Profit factor:** `1.04` +> **Max Drawdown:** `9.23% (0.01255 BTC)` The relative profit of `1.2%` is the average profit per trade. The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. +Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. +Max drawdown corresponds to the backtesting metric `Absolute Drawdown (Account)` - calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`. ### /forceexit From 8c46d19071d96dae2eca4800e982b0e668e461d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:27:54 +0200 Subject: [PATCH 10/25] Fix backtesting bug balance was never released on cancelled trades --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f7d92081f..6eeefbfac 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1055,6 +1055,7 @@ class Backtesting: # Close trade open_trade_count -= 1 open_trades[pair].remove(t) + LocalTrade.trades_open.remove(t) self.wallets.update() # 2. Process entries. From 53bfa7931d1e4ac02a76a5b9be17d4fd6c55725a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:32:22 +0200 Subject: [PATCH 11/25] Add rudimentary test for prior bug Test fails without the fix in 8c46d19071d96dae2eca4800e982b0e668e461d6 --- tests/optimize/test_backtest_detail.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 4b4c446e0..a18196507 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -7,6 +7,7 @@ import pytest from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType from freqtrade.optimize.backtesting import Backtesting +from freqtrade.persistence.trade_model import LocalTrade from tests.conftest import patch_exchange from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset, tests_timeframe) @@ -964,5 +965,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) assert res.open_date == _get_frame_time_from_offset(trade.open_tick) assert res.close_date == _get_frame_time_from_offset(trade.close_tick) assert res.is_short == trade.is_short + assert len(LocalTrade.trades) == len(data.trades) + assert len(LocalTrade.trades_open) == 0 backtesting.cleanup() del backtesting From 474e6705e622aa7372c7a9b203df8553dbdcded3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:27:43 +0200 Subject: [PATCH 12/25] Add Profit factor to backtesting --- docs/backtesting.md | 4 ++++ docs/telegram-usage.md | 6 +++--- freqtrade/optimize/optimize_reports.py | 6 ++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 76718d206..50fc96923 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -300,6 +300,7 @@ A backtesting result will look like that: | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | | CAGR % | 460.87% | +| Profit factor | 1.11 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -399,6 +400,7 @@ It contains some useful key metrics about performance of your strategy on backte | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | | CAGR % | 460.87% | +| Profit factor | 1.11 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -444,6 +446,8 @@ It contains some useful key metrics about performance of your strategy on backte - `Final balance`: Final balance - starting balance + absolute profit. - `Absolute profit`: Profit made in stake currency. - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. +- `CAGR %`: Compound annual growth rate. +- `Profit factor`: profit / loss. - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. - `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 95e7eaa16..2145797b4 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -275,9 +275,9 @@ Return a summary of your profit/loss and performance. > **Max Drawdown:** `9.23% (0.01255 BTC)` The relative profit of `1.2%` is the average profit per trade. -The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. -Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. -Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. +The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. +Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. +Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. Max drawdown corresponds to the backtesting metric `Absolute Drawdown (Account)` - calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`. ### /forceexit diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 79cb8a2bd..44ac4a5b3 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -416,6 +416,9 @@ def generate_strategy_stats(pairlist: List[str], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + winning_profit = results.loc[results['profit_abs'] > 0, 'profit_abs'].sum() + losing_profit = results.loc[results['profit_abs'] < 0, 'profit_abs'].sum() + profit_factor = winning_profit / abs(losing_profit) if losing_profit else 0.0 backtest_days = (max_date - min_date).days or 1 strat_stats = { @@ -443,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str], 'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(), 'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(), 'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']), + 'profit_factor': profit_factor, 'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT), 'backtest_start_ts': int(min_date.timestamp() * 1000), 'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT), @@ -779,6 +783,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: strat_results['stake_currency'])), ('Total profit %', f"{strat_results['profit_total']:.2%}"), ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), + ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor' + in strat_results else 'N/A'), ('Trades per day', strat_results['trades_per_day']), ('Avg. daily profit %', f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), From 0168343b7656eb572c0bda50daa774e2f9549e5f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:53:25 +0200 Subject: [PATCH 13/25] Add trading-volume to api schema --- freqtrade/persistence/trade_model.py | 7 +++---- freqtrade/rpc/api_server/api_schemas.py | 1 + tests/conftest_trades.py | 14 ++++++++++++++ tests/rpc/test_rpc_apiserver.py | 7 ++++--- tests/test_persistence.py | 1 + 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3a52c0660..39ebd75b4 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1363,8 +1363,7 @@ class Trade(_DECL_BASE, LocalTrade): trading_volume = Order.query.with_entities( func.sum(Order.cost).label('volume') ).filter( - (Order.order_filled_date >= start_date) - & (Order.status == 'closed') - ) \ - .scalar() + Order.order_filled_date >= start_date, + Order.status == 'closed' + ).scalar() return trading_volume diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index fda2d7ea0..7566e2ac0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -107,6 +107,7 @@ class Profit(BaseModel): profit_factor: float max_drawdown: float max_drawdown_abs: float + trading_volume: Optional[float] class SellReason(BaseModel): diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 006eab98f..1a8cf3183 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -29,6 +29,7 @@ def mock_order_1(is_short: bool): 'average': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -65,6 +66,7 @@ def mock_order_2(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -79,6 +81,7 @@ def mock_order_2_sell(is_short: bool): 'price': 0.128, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -126,6 +129,7 @@ def mock_order_3(is_short: bool): 'price': 0.05, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -141,6 +145,7 @@ def mock_order_3_sell(is_short: bool): 'average': 0.06, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -186,6 +191,7 @@ def mock_order_4(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 0.0, + 'cost': 15.129, 'remaining': 123.0, } @@ -225,6 +231,7 @@ def mock_order_5(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -239,6 +246,7 @@ def mock_order_5_stoploss(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 0.0, + 'cost': 0.0, 'remaining': 123.0, } @@ -281,6 +289,7 @@ def mock_order_6(is_short: bool): 'price': 0.15, 'amount': 2.0, 'filled': 2.0, + 'cost': 0.3, 'remaining': 0.0, } @@ -295,6 +304,7 @@ def mock_order_6_sell(is_short: bool): 'price': 0.15 if is_short else 0.20, 'amount': 2.0, 'filled': 0.0, + 'cost': 0.0, 'remaining': 2.0, } @@ -337,6 +347,7 @@ def short_order(): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -351,6 +362,7 @@ def exit_short_order(): 'price': 0.128, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.744, 'remaining': 0.0, } @@ -424,6 +436,7 @@ def leverage_order(): 'amount': 123.0, 'filled': 123.0, 'remaining': 0.0, + 'cost': 15.129, 'leverage': 5.0 } @@ -439,6 +452,7 @@ def leverage_order_sell(): 'amount': 123.0, 'filled': 123.0, 'remaining': 0.0, + 'cost': 15.744, 'leverage': 5.0 } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index afbc92c5d..b0ff5e1b2 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -725,7 +725,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2, - 'profit_factor': 0.0, + 'profit_factor': 0.0, 'trading_volume': 91.074, } ), ( @@ -740,7 +740,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0, - 'profit_factor': None, + 'profit_factor': None, 'trading_volume': 91.074, } ), ( @@ -755,7 +755,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1, - 'profit_factor': 0.02775724835771106, + 'profit_factor': 0.02775724835771106, 'trading_volume': 91.074, } ) ]) @@ -812,6 +812,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'profit_factor': expected['profit_factor'], 'max_drawdown': ANY, 'max_drawdown_abs': ANY, + 'trading_volume': expected['trading_volume'], } diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8c12d2ea0..357233dfa 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -2269,6 +2269,7 @@ def test_Trade_object_idem(): 'get_exit_reason_performance', 'get_enter_tag_performance', 'get_mix_tag_performance', + 'get_trading_volume', ) From 0809f9aef69776805912bf8b32621a0a3d481959 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 17:44:15 +0200 Subject: [PATCH 14/25] Add offset to trade response --- freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/rpc.py | 1 + tests/rpc/test_rpc_apiserver.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 7566e2ac0..333f2fe6e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -283,6 +283,7 @@ class OpenTradeSchema(TradeSchema): class TradeResponse(BaseModel): trades: List[TradeSchema] trades_count: int + offset: int total_trades: int diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 31fe4c469..dbbb78c98 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -365,6 +365,7 @@ class RPC: return { "trades": output, "trades_count": len(output), + "offset": offset, "total_trades": Trade.get_trades([Trade.is_open.is_(False)]).count(), } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b0ff5e1b2..c0de54c6d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -578,9 +578,10 @@ def test_api_trades(botclient, mocker, fee, markets, is_short): ) rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) - assert len(rc.json()) == 3 + assert len(rc.json()) == 4 assert rc.json()['trades_count'] == 0 assert rc.json()['total_trades'] == 0 + assert rc.json()['offset'] == 0 create_mock_trades(fee, is_short=is_short) Trade.query.session.flush() From 0d967f93baf69e08bda264df0702f4a433abf64b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jun 2022 16:13:00 +0200 Subject: [PATCH 15/25] Improve performance of some RPC calls These don't need orders to be loaded. As a side-effect, this will also reduce the strain on the database. --- freqtrade/persistence/trade_model.py | 13 +++++++++---- freqtrade/rpc/rpc.py | 5 +++-- tests/test_persistence.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 39ebd75b4..0c8c985c8 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func) -from sqlalchemy.orm import Query, relationship +from sqlalchemy.orm import Query, lazyload, relationship from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort from freqtrade.enums import ExitType, TradingMode @@ -1115,7 +1115,7 @@ class Trade(_DECL_BASE, LocalTrade): ) @staticmethod - def get_trades(trade_filter=None) -> Query: + def get_trades(trade_filter=None, include_orders: bool = True) -> Query: """ Helper function to query Trades using filters. NOTE: Not supported in Backtesting. @@ -1130,9 +1130,14 @@ class Trade(_DECL_BASE, LocalTrade): if trade_filter is not None: if not isinstance(trade_filter, list): trade_filter = [trade_filter] - return Trade.query.filter(*trade_filter) + this_query = Trade.query.filter(*trade_filter) else: - return Trade.query + this_query = Trade.query + if not include_orders: + # Don't load order relations + # Consider using noload or raiseload instead of lazyload + this_query = this_query.options(lazyload(Trade.orders)) + return this_query @staticmethod def get_open_order_trades() -> List['Trade']: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dbbb78c98..c42a6f683 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -380,7 +380,7 @@ class RPC: return 'losses' else: return 'draws' - trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)]) + trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False) # Sell reason exit_reasons = {} for trade in trades: @@ -408,7 +408,8 @@ class RPC: """ Returns cumulative profit statistics """ trade_filter = ((Trade.is_open.is_(False) & (Trade.close_date >= start_date)) | Trade.is_open.is_(True)) - trades: List[Trade] = Trade.get_trades(trade_filter).order_by(Trade.id).all() + trades: List[Trade] = Trade.get_trades( + trade_filter, include_orders=False).order_by(Trade.id).all() profit_all_coin = [] profit_all_ratio = [] diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 357233dfa..deaad258b 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -2075,6 +2075,24 @@ def test_get_trades_proxy(fee, use_db, is_short): Trade.use_db = True +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize('is_short', [True, False]) +def test_get_trades__query(fee, is_short): + query = Trade.get_trades([]) + # without orders there should be no join issued. + query1 = Trade.get_trades([], include_orders=False) + + assert "JOIN orders" in str(query) + assert "JOIN orders" not in str(query1) + + create_mock_trades(fee, is_short) + query = Trade.get_trades([]) + query1 = Trade.get_trades([], include_orders=False) + + assert "JOIN orders" in str(query) + assert "JOIN orders" not in str(query1) + + def test_get_trades_backtest(): Trade.use_db = False with pytest.raises(NotImplementedError, match=r"`Trade.get_trades\(\)` not .*"): From 8406010260e3d79be0638b375b63fd447d4711da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 03:01:26 +0000 Subject: [PATCH 16/25] Bump types-cachetools from 5.0.1 to 5.0.2 Bumps [types-cachetools](https://github.com/python/typeshed) from 5.0.1 to 5.0.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 19912d59c..53c85f176 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ time-machine==2.7.0 nbconvert==6.5.0 # mypy types -types-cachetools==5.0.1 +types-cachetools==5.0.2 types-filelock==3.2.7 types-requests==2.27.30 types-tabulate==0.8.9 From 55fb7656dfdafebf754d6a5b14c8bdd0eca2eb29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 06:58:41 +0200 Subject: [PATCH 17/25] Update pre-commit cachetools --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5c1a36f5..e057627cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: mypy exclude: build_helpers additional_dependencies: - - types-cachetools==5.0.1 + - types-cachetools==5.0.2 - types-filelock==3.2.7 - types-requests==2.27.30 - types-tabulate==0.8.9 From 0804fc7a3af91c25b291bd991132cf77d8d42946 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:01:35 +0200 Subject: [PATCH 18/25] CI should run ccxt tests only once --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 551268af7..2f67ec5fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: - name: Tests incl. ccxt compatibility tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun - if: matrix.python-version == '3.9' + if: matrix.python-version == '3.9' and matrix.os == 'ubuntu-22.04' - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.9') From f9668ede4a54edda390197b3261504b3d526c77d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:02:12 +0200 Subject: [PATCH 19/25] Fix CI Syntax error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f67ec5fe..81a18c4c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: - name: Tests incl. ccxt compatibility tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun - if: matrix.python-version == '3.9' and matrix.os == 'ubuntu-22.04' + if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-22.04' - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.9') From 50c19ece53644808adf80a95eebefdf6fe3f4c6d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:05:51 +0200 Subject: [PATCH 20/25] Fix ccxt test gateio flukyness --- tests/exchange/test_ccxt_compat.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index e016873cb..50154bcaf 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -199,8 +199,13 @@ class TestCCXTExchange(): l2 = exchange.fetch_l2_order_book(pair) assert 'asks' in l2 assert 'bids' in l2 + assert len(l2['asks']) >= 1 + assert len(l2['bids']) >= 1 l2_limit_range = exchange._ft_has['l2_limit_range'] l2_limit_range_required = exchange._ft_has['l2_limit_range_required'] + if exchangename == 'gateio': + # TODO: Gateio is unstable here at the moment, ignoring the limit partially. + return for val in [1, 2, 5, 25, 100]: l2 = exchange.fetch_l2_order_book(pair, val) if not l2_limit_range or val in l2_limit_range: From 996372b8f6c18b9721034bd6abfd41532f3e2b62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 05:06:39 +0000 Subject: [PATCH 21/25] Bump colorama from 0.4.4 to 0.4.5 Bumps [colorama](https://github.com/tartley/colorama) from 0.4.4 to 0.4.5. - [Release notes](https://github.com/tartley/colorama/releases) - [Changelog](https://github.com/tartley/colorama/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tartley/colorama/compare/0.4.4...0.4.5) --- updated-dependencies: - dependency-name: colorama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b2dbd921e..bd28e3768 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ aiofiles==0.8.0 psutil==5.9.1 # Support for colorized terminal output -colorama==0.4.4 +colorama==0.4.5 # Building config files interactively questionary==1.10.0 prompt-toolkit==3.0.29 From e1e3a903f98aebd0d2db6b9c6e0e5f51b70075c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 05:07:35 +0000 Subject: [PATCH 22/25] Bump ccxt from 1.87.12 to 1.88.15 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.87.12 to 1.88.15. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.87.12...1.88.15) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b2dbd921e..ec244806d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.87.12 +ccxt==1.88.15 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 1cd2b0504a9e1c684326ab59a6dcb6f2a8eb85a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:15:15 +0200 Subject: [PATCH 23/25] Run regular tests for 3.9 under other ubuntu systems --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81a18c4c9..818d250ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc - if: matrix.python-version != '3.9' + if: matrix.python-version != '3.9' && matrix.os != 'ubuntu-22.04' - name: Tests incl. ccxt compatibility tests run: | From 3189b284c014c4624ba27fc8abbde982c5b36c0c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 08:04:34 +0200 Subject: [PATCH 24/25] Fix tests condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 818d250ca..4fe1ad853 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc - if: matrix.python-version != '3.9' && matrix.os != 'ubuntu-22.04' + if: matrix.python-version != '3.9' || matrix.os != 'ubuntu-22.04' - name: Tests incl. ccxt compatibility tests run: | From 15fac746a8129e00acb86246e16b00fb7daea89f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 06:59:58 +0000 Subject: [PATCH 25/25] Bump mkdocs-material from 8.3.4 to 8.3.6 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.3.4 to 8.3.6. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.3.4...8.3.6) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 1f342ca02..6477ad23f 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.3.4 +mkdocs-material==8.3.6 mdx_truly_sane_lists==1.2 pymdown-extensions==9.5 jinja2==3.1.2