From 28411da83eedd4d0441e1c8f5836d0f3a0864e39 Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Tue, 22 Sep 2020 22:28:12 +0100 Subject: [PATCH 01/32] Add the telegram command function template. --- freqtrade/rpc/telegram.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a01efaed6..a2dae387e 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -108,6 +108,7 @@ class Telegram(RPC): CommandHandler('edge', self._edge), CommandHandler('help', self._help), CommandHandler('version', self._version), + CommandHandler('stats', self._stats), ] for handle in handles: self._updater.dispatcher.add_handler(handle) @@ -738,6 +739,19 @@ class Telegram(RPC): """ self._send_msg('*Version:* `{}`'.format(__version__)) + @authorized_only + def _stats(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /stats + https://github.com/freqtrade/freqtrade/issues/3783 + Show stats of recent trades + :param update: message update + :return: None + """ + # TODO: self._send_msg(...) + trades = self._rpc_trade_history(-1) + + @authorized_only def _show_config(self, update: Update, context: CallbackContext) -> None: """ From 44ad0f631c00ed00063fa61ebe3aefd6b5f736a1 Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Sat, 26 Sep 2020 22:40:54 +0100 Subject: [PATCH 02/32] Summarize trade reason for telegram command /stats. --- freqtrade/rpc/telegram.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a2dae387e..47e9d67dc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -749,9 +749,40 @@ class Telegram(RPC): :return: None """ # TODO: self._send_msg(...) - trades = self._rpc_trade_history(-1) - + def trade_win_loss(trade): + if trade['profit_abs'] > 0: + return 'Wins' + elif trade['profit_abs'] < 0: + return 'Losses' + else: + return 'Draws' + trades = self._rpc_trade_history(-1) + trades_closed = [trade for trade in trades if not trade['is_open']] + + # Sell reason + sell_reasons = {} + for trade in trades_closed: + if trade['sell_reason'] in sell_reasons: + sell_reasons[trade['sell_reason']][trade_win_loss(trade)] += 1 + else: + win_loss_count = {'Wins': 0, 'Losses': 0, 'Draws': 0} + win_loss_count[trade_win_loss(trade)] += 1 + sell_reasons[trade['sell_reason']] = win_loss_count + sell_reason_msg = [ + '| Sell Reason | Sells | Wins | Draws | Losses |', + '|-------------|------:|-----:|------:|-------:|' + ] + # | Sell Reason | Sells | Wins | Draws | Losses | + # |-------------|------:|-----:|------:|-------:| + # | test | 1 | 2 | 3 | 4 | + for reason, count in sell_reasons.items(): + msg = f'| `{reason}` | `{sum(count.values())}` | `{count['Wins']}` | `{count['Draws']}` | `{count['Losses']}` |' + sell_reason_msg.append(msg) + + # TODO: Duration + + @authorized_only def _show_config(self, update: Update, context: CallbackContext) -> None: """ From 627e221b654e5b0ebf87d6f299f70e26c798e3b7 Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Sun, 27 Sep 2020 20:23:13 +0100 Subject: [PATCH 03/32] Use tabulate to create sell reason message. --- freqtrade/rpc/telegram.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 47e9d67dc..ea8597469 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -769,16 +769,21 @@ class Telegram(RPC): win_loss_count = {'Wins': 0, 'Losses': 0, 'Draws': 0} win_loss_count[trade_win_loss(trade)] += 1 sell_reasons[trade['sell_reason']] = win_loss_count - sell_reason_msg = [ - '| Sell Reason | Sells | Wins | Draws | Losses |', - '|-------------|------:|-----:|------:|-------:|' - ] + sell_reasons_tabulate = [] # | Sell Reason | Sells | Wins | Draws | Losses | # |-------------|------:|-----:|------:|-------:| # | test | 1 | 2 | 3 | 4 | for reason, count in sell_reasons.items(): - msg = f'| `{reason}` | `{sum(count.values())}` | `{count['Wins']}` | `{count['Draws']}` | `{count['Losses']}` |' - sell_reason_msg.append(msg) + sell_reasons_tabulate.append([ + reason, sum(count.values()), + count['Wins'], + count['Draws'], + count['Losses'] + ]) + sell_reasons_msg = tabulate( + sell_reasons_tabulate, + headers=['Sell Reason', 'Sells', 'Wins', 'Draws', 'Losses'] + ) # TODO: Duration From 7bce2cd29daa65a8013d0f2f44fee817901b2465 Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Mon, 28 Sep 2020 20:30:20 +0100 Subject: [PATCH 04/32] Add trade duration by win/loss. --- freqtrade/rpc/telegram.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ea8597469..bfe486951 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -763,16 +763,10 @@ class Telegram(RPC): # Sell reason sell_reasons = {} for trade in trades_closed: - if trade['sell_reason'] in sell_reasons: - sell_reasons[trade['sell_reason']][trade_win_loss(trade)] += 1 - else: - win_loss_count = {'Wins': 0, 'Losses': 0, 'Draws': 0} - win_loss_count[trade_win_loss(trade)] += 1 - sell_reasons[trade['sell_reason']] = win_loss_count + if trade['sell_reason'] not in sell_reasons: + sell_reasons[trade['sell_reason']] = {'Wins': 0, 'Losses': 0, 'Draws': 0} + sell_reasons[trade['sell_reason']][trade_win_loss(trade)] += 1 sell_reasons_tabulate = [] - # | Sell Reason | Sells | Wins | Draws | Losses | - # |-------------|------:|-----:|------:|-------:| - # | test | 1 | 2 | 3 | 4 | for reason, count in sell_reasons.items(): sell_reasons_tabulate.append([ reason, sum(count.values()), @@ -785,9 +779,22 @@ class Telegram(RPC): headers=['Sell Reason', 'Sells', 'Wins', 'Draws', 'Losses'] ) - # TODO: Duration + # Duration + dur = {'Wins': [], 'Draws': [], 'Losses': []} + for trade in trades_closed: + if trade['close_date'] is not None and trade['open_date'] is not None: + trade_dur = arrow.get(trade['close_date']) - arrow.get(trade['open_date']) + dur[trade_win_loss(trade)].append(trade_dur) + wins_dur = sum(dur['Wins']) / len(dur['Wins']) if len(dur['Wins']) > 0 else 'N/A' + draws_dur = sum(dur['Draws']) / len(dur['Draws']) if len(dur['Draws']) > 0 else 'N/A' + losses_dur = sum(dur['Losses']) / len(dur['Losses']) if len(dur['Losses']) > 0 else 'N/A' + duration_msg = tabulate( + [['Wins', str(wins_dur)], ['Draws', str(draws_dur)], ['Losses', str(losses_dur)]], + headers=['', 'Duration'] + ) + + self._send_msg('\n'.join([sell_reasons_msg, duration_msg])) - @authorized_only def _show_config(self, update: Update, context: CallbackContext) -> None: """ From 355afc082e4619e7de11640bae4b6dfc8cc61f81 Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Mon, 5 Oct 2020 10:05:15 +0100 Subject: [PATCH 05/32] Add command 'stats' in expected test output. --- tests/rpc/test_rpc_telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 762780111..bcb9abc85 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -78,7 +78,8 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], ['trades'], " "['delete'], ['performance'], ['daily'], ['count'], ['reload_config', " "'reload_conf'], ['show_config', 'show_conf'], ['stopbuy'], " - "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version']]") + "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version'], " + "['stats']]") assert log_has(message_str, caplog) From 1c27aaab724b781487a92f103363071d1276c18f Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Sun, 18 Oct 2020 20:24:13 +0100 Subject: [PATCH 06/32] Declare type of 'dur'. --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index bfe486951..8404625f1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -6,7 +6,7 @@ This module manage Telegram communication import json import logging import arrow -from typing import Any, Callable, Dict +from typing import Any, Callable, Dict, List from tabulate import tabulate from telegram import ParseMode, ReplyKeyboardMarkup, Update @@ -780,7 +780,7 @@ class Telegram(RPC): ) # Duration - dur = {'Wins': [], 'Draws': [], 'Losses': []} + dur: Dict[str, List[int]] = {'Wins': [], 'Draws': [], 'Losses': []} for trade in trades_closed: if trade['close_date'] is not None and trade['open_date'] is not None: trade_dur = arrow.get(trade['close_date']) - arrow.get(trade['open_date']) From c556d1b37e6e6367e1c05362522e494df274275b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Dec 2020 14:06:46 +0100 Subject: [PATCH 07/32] Make /stats working --- freqtrade/rpc/rpc.py | 4 ++++ freqtrade/rpc/telegram.py | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9ac271ba0..e17ee6b4f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -275,6 +275,10 @@ class RPC: "trades_count": len(output) } + def _rpc_stats(self): + trades = trades = Trade.get_trades([Trade.is_open.is_(False)]) + return trades + def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: """ Returns cumulative profit statistics """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 074a6367f..29d2c6a01 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -782,22 +782,22 @@ class Telegram(RPC): """ # TODO: self._send_msg(...) def trade_win_loss(trade): - if trade['profit_abs'] > 0: + if trade.close_profit_abs > 0: return 'Wins' - elif trade['profit_abs'] < 0: + elif trade.close_profit_abs < 0: return 'Losses' else: return 'Draws' - trades = self._rpc_trade_history(-1) - trades_closed = [trade for trade in trades if not trade['is_open']] + trades = self._rpc_stats() + trades_closed = [trade for trade in trades if not trade.is_open] # Sell reason sell_reasons = {} for trade in trades_closed: - if trade['sell_reason'] not in sell_reasons: - sell_reasons[trade['sell_reason']] = {'Wins': 0, 'Losses': 0, 'Draws': 0} - sell_reasons[trade['sell_reason']][trade_win_loss(trade)] += 1 + if trade.sell_reason not in sell_reasons: + sell_reasons[trade.sell_reason] = {'Wins': 0, 'Losses': 0, 'Draws': 0} + sell_reasons[trade.sell_reason][trade_win_loss(trade)] += 1 sell_reasons_tabulate = [] for reason, count in sell_reasons.items(): sell_reasons_tabulate.append([ @@ -814,8 +814,8 @@ class Telegram(RPC): # Duration dur: Dict[str, List[int]] = {'Wins': [], 'Draws': [], 'Losses': []} for trade in trades_closed: - if trade['close_date'] is not None and trade['open_date'] is not None: - trade_dur = arrow.get(trade['close_date']) - arrow.get(trade['open_date']) + if trade.close_date is not None and trade.open_date is not None: + trade_dur = (trade.close_date - trade.open_date).total_seconds() dur[trade_win_loss(trade)].append(trade_dur) wins_dur = sum(dur['Wins']) / len(dur['Wins']) if len(dur['Wins']) > 0 else 'N/A' draws_dur = sum(dur['Draws']) / len(dur['Draws']) if len(dur['Draws']) > 0 else 'N/A' @@ -824,8 +824,9 @@ class Telegram(RPC): [['Wins', str(wins_dur)], ['Draws', str(draws_dur)], ['Losses', str(losses_dur)]], headers=['', 'Duration'] ) + msg = (f"""```{sell_reasons_msg}```\n```{duration_msg}```""") - self._send_msg('\n'.join([sell_reasons_msg, duration_msg])) + self._send_msg(msg, ParseMode.MARKDOWN) @authorized_only def _show_config(self, update: Update, context: CallbackContext) -> None: From 143423145cacec63868390045c1e911390dba327 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Dec 2020 14:38:42 +0100 Subject: [PATCH 08/32] Refactor most of the logic to rpc.py this way /stats can be used by other RPC methods too --- freqtrade/rpc/rpc.py | 31 +++++++++++++++++++- freqtrade/rpc/telegram.py | 59 ++++++++++++++++----------------------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index e17ee6b4f..d7a59390d 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -276,8 +276,37 @@ class RPC: } def _rpc_stats(self): + """ + Generate generic stats for trades in database + """ + def trade_win_loss(trade): + if trade.close_profit_abs > 0: + return 'Wins' + elif trade.close_profit_abs < 0: + return 'Losses' + else: + return 'Draws' trades = trades = Trade.get_trades([Trade.is_open.is_(False)]) - return trades + # Sell reason + sell_reasons = {} + for trade in trades: + if trade.sell_reason not in sell_reasons: + sell_reasons[trade.sell_reason] = {'Wins': 0, 'Losses': 0, 'Draws': 0} + sell_reasons[trade.sell_reason][trade_win_loss(trade)] += 1 + + # Duration + dur: Dict[str, List[int]] = {'Wins': [], 'Draws': [], 'Losses': []} + for trade in trades: + if trade.close_date is not None and trade.open_date is not None: + trade_dur = (trade.close_date - trade.open_date).total_seconds() + dur[trade_win_loss(trade)].append(trade_dur) + + wins_dur = sum(dur['Wins']) / len(dur['Wins']) if len(dur['Wins']) > 0 else 'N/A' + draws_dur = sum(dur['Draws']) / len(dur['Draws']) if len(dur['Draws']) > 0 else 'N/A' + losses_dur = sum(dur['Losses']) / len(dur['Losses']) if len(dur['Losses']) > 0 else 'N/A' + + durations = {'wins': wins_dur, 'draws': draws_dur, 'losses': losses_dur} + return sell_reasons, durations def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 29d2c6a01..7c7007f86 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -3,6 +3,7 @@ """ This module manage Telegram communication """ +from datetime import timedelta import json import logging from typing import Any, Callable, Dict, List, Union @@ -775,56 +776,44 @@ class Telegram(RPC): def _stats(self, update: Update, context: CallbackContext) -> None: """ Handler for /stats - https://github.com/freqtrade/freqtrade/issues/3783 Show stats of recent trades - :param update: message update :return: None """ - # TODO: self._send_msg(...) - def trade_win_loss(trade): - if trade.close_profit_abs > 0: - return 'Wins' - elif trade.close_profit_abs < 0: - return 'Losses' - else: - return 'Draws' + sell_reasons, durations = self._rpc_stats() - trades = self._rpc_stats() - trades_closed = [trade for trade in trades if not trade.is_open] - - # Sell reason - sell_reasons = {} - for trade in trades_closed: - if trade.sell_reason not in sell_reasons: - sell_reasons[trade.sell_reason] = {'Wins': 0, 'Losses': 0, 'Draws': 0} - sell_reasons[trade.sell_reason][trade_win_loss(trade)] += 1 sell_reasons_tabulate = [] + reason_map = { + 'roi': 'ROI', + 'stop_loss': 'Stoploss', + 'trailing_stop_loss': 'Trail. Stop', + 'stoploss_on_exchange': 'Stoploss', + 'sell_signal': 'Sell Signal', + 'force_sell': 'Forcesell', + 'emergency_sell': 'Emergency Sell', + } for reason, count in sell_reasons.items(): sell_reasons_tabulate.append([ - reason, sum(count.values()), + reason_map.get(reason, reason), + sum(count.values()), count['Wins'], - count['Draws'], + # count['Draws'], count['Losses'] ]) sell_reasons_msg = tabulate( sell_reasons_tabulate, - headers=['Sell Reason', 'Sells', 'Wins', 'Draws', 'Losses'] + headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] ) - # Duration - dur: Dict[str, List[int]] = {'Wins': [], 'Draws': [], 'Losses': []} - for trade in trades_closed: - if trade.close_date is not None and trade.open_date is not None: - trade_dur = (trade.close_date - trade.open_date).total_seconds() - dur[trade_win_loss(trade)].append(trade_dur) - wins_dur = sum(dur['Wins']) / len(dur['Wins']) if len(dur['Wins']) > 0 else 'N/A' - draws_dur = sum(dur['Draws']) / len(dur['Draws']) if len(dur['Draws']) > 0 else 'N/A' - losses_dur = sum(dur['Losses']) / len(dur['Losses']) if len(dur['Losses']) > 0 else 'N/A' - duration_msg = tabulate( - [['Wins', str(wins_dur)], ['Draws', str(draws_dur)], ['Losses', str(losses_dur)]], - headers=['', 'Duration'] + duration_msg = tabulate([ + ['Wins', str(timedelta(seconds=durations['wins'])) + if durations['wins'] != 'N/A' else 'N/A'], + # ['Draws', str(timedelta(seconds=durations['draws']))], + ['Losses', str(timedelta(seconds=durations['losses'])) + if durations['losses'] != 'N/A' else 'N/A'] + ], + headers=['', 'Avg. Duration'] ) - msg = (f"""```{sell_reasons_msg}```\n```{duration_msg}```""") + msg = (f"""```\n{sell_reasons_msg}```\n```\n{duration_msg}```""") self._send_msg(msg, ParseMode.MARKDOWN) From aa27c9ace2fa3ac9b83780de5f1d0e4a9bd70fbf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Dec 2020 14:39:50 +0100 Subject: [PATCH 09/32] Reorder methods in telegram /stats is closely related to /profit --- freqtrade/rpc/telegram.py | 90 +++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7c7007f86..76d9292b4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -390,6 +390,51 @@ class Telegram(RPC): f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") self._send_msg(markdown_msg) + @authorized_only + def _stats(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /stats + Show stats of recent trades + :return: None + """ + sell_reasons, durations = self._rpc_stats() + + sell_reasons_tabulate = [] + reason_map = { + 'roi': 'ROI', + 'stop_loss': 'Stoploss', + 'trailing_stop_loss': 'Trail. Stop', + 'stoploss_on_exchange': 'Stoploss', + 'sell_signal': 'Sell Signal', + 'force_sell': 'Forcesell', + 'emergency_sell': 'Emergency Sell', + } + for reason, count in sell_reasons.items(): + sell_reasons_tabulate.append([ + reason_map.get(reason, reason), + sum(count.values()), + count['Wins'], + # count['Draws'], + count['Losses'] + ]) + sell_reasons_msg = tabulate( + sell_reasons_tabulate, + headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] + ) + + duration_msg = tabulate([ + ['Wins', str(timedelta(seconds=durations['wins'])) + if durations['wins'] != 'N/A' else 'N/A'], + # ['Draws', str(timedelta(seconds=durations['draws']))], + ['Losses', str(timedelta(seconds=durations['losses'])) + if durations['losses'] != 'N/A' else 'N/A'] + ], + headers=['', 'Avg. Duration'] + ) + msg = (f"""```\n{sell_reasons_msg}```\n```\n{duration_msg}```""") + + self._send_msg(msg, ParseMode.MARKDOWN) + @authorized_only def _balance(self, update: Update, context: CallbackContext) -> None: """ Handler for /balance """ @@ -772,51 +817,6 @@ class Telegram(RPC): """ self._send_msg('*Version:* `{}`'.format(__version__)) - @authorized_only - def _stats(self, update: Update, context: CallbackContext) -> None: - """ - Handler for /stats - Show stats of recent trades - :return: None - """ - sell_reasons, durations = self._rpc_stats() - - sell_reasons_tabulate = [] - reason_map = { - 'roi': 'ROI', - 'stop_loss': 'Stoploss', - 'trailing_stop_loss': 'Trail. Stop', - 'stoploss_on_exchange': 'Stoploss', - 'sell_signal': 'Sell Signal', - 'force_sell': 'Forcesell', - 'emergency_sell': 'Emergency Sell', - } - for reason, count in sell_reasons.items(): - sell_reasons_tabulate.append([ - reason_map.get(reason, reason), - sum(count.values()), - count['Wins'], - # count['Draws'], - count['Losses'] - ]) - sell_reasons_msg = tabulate( - sell_reasons_tabulate, - headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] - ) - - duration_msg = tabulate([ - ['Wins', str(timedelta(seconds=durations['wins'])) - if durations['wins'] != 'N/A' else 'N/A'], - # ['Draws', str(timedelta(seconds=durations['draws']))], - ['Losses', str(timedelta(seconds=durations['losses'])) - if durations['losses'] != 'N/A' else 'N/A'] - ], - headers=['', 'Avg. Duration'] - ) - msg = (f"""```\n{sell_reasons_msg}```\n```\n{duration_msg}```""") - - self._send_msg(msg, ParseMode.MARKDOWN) - @authorized_only def _show_config(self, update: Update, context: CallbackContext) -> None: """ From 245c19f5e9aff5a797e0f1d71924d552b1f86a1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Dec 2020 14:48:56 +0100 Subject: [PATCH 10/32] Add simple test for /stats call --- freqtrade/rpc/rpc.py | 4 ++-- tests/conftest_trades.py | 2 ++ tests/rpc/test_rpc_telegram.py | 35 ++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index d7a59390d..c4b4117ff 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -280,9 +280,9 @@ class RPC: Generate generic stats for trades in database """ def trade_win_loss(trade): - if trade.close_profit_abs > 0: + if trade.close_profit > 0: return 'Wins' - elif trade.close_profit_abs < 0: + elif trade.close_profit < 0: return 'Losses' else: return 'Draws' diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 78388f022..fac822b2b 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -82,6 +82,7 @@ def mock_trade_2(fee): is_open=False, open_order_id='dry_run_sell_12345', strategy='DefaultStrategy', + sell_reason='sell_signal' ) o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy') trade.orders.append(o) @@ -134,6 +135,7 @@ def mock_trade_3(fee): close_profit=0.01, exchange='bittrex', is_open=False, + sell_reason='roi' ) o = Order.parse_from_ccxt_object(mock_order_3(), 'XRP/BTC', 'buy') trade.orders.append(o) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 73a549860..725c1411e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -469,6 +469,41 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, assert '*Best Performing:* `ETH/BTC: 6.20%`' in msg_mock.call_args_list[-1][0][0] +def test_telegram_stats(default_conf, update, ticker, ticker_sell_up, fee, + limit_buy_order, limit_sell_order, mocker) -> None: + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_fee=fee, + ) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) + + telegram._stats(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + # assert 'No trades yet.' in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + + # Create some test data + create_mock_trades(fee) + + telegram._stats(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + assert 'Sell Reason' in msg_mock.call_args_list[-1][0][0] + assert 'ROI' in msg_mock.call_args_list[-1][0][0] + assert 'Avg. Duration' in msg_mock.call_args_list[-1][0][0] + msg_mock.reset_mock() + + def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tickers) -> None: default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) From 0c0eb8236d24f84efaad4b51bd768ff23ea462f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Dec 2020 05:48:23 +0000 Subject: [PATCH 11/32] Bump mkdocs-material from 6.1.6 to 6.1.7 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 6.1.6 to 6.1.7. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/6.1.6...6.1.7) Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 87bc6dfdd..2b133cb07 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,3 @@ -mkdocs-material==6.1.6 +mkdocs-material==6.1.7 mdx_truly_sane_lists==1.2 pymdown-extensions==8.0.1 From 647e6509a477b9b07b7e9f9d73ee3ad756d2b499 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Dec 2020 05:48:43 +0000 Subject: [PATCH 12/32] Bump ccxt from 1.38.55 to 1.38.87 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.38.55 to 1.38.87. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.38.55...1.38.87) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f59754f93..105839f0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.19.4 pandas==1.1.4 -ccxt==1.38.55 +ccxt==1.38.87 aiohttp==3.7.3 SQLAlchemy==1.3.20 python-telegram-bot==13.1 From e873cafdc49d46c2398550a77bd29dd61816a050 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 14:54:39 +0100 Subject: [PATCH 13/32] Beautify code a bit --- freqtrade/rpc/rpc.py | 20 ++++++++++---------- freqtrade/rpc/telegram.py | 17 +++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c4b4117ff..49e5bc2d2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -275,38 +275,38 @@ class RPC: "trades_count": len(output) } - def _rpc_stats(self): + def _rpc_stats(self) -> Dict[str, Any]: """ Generate generic stats for trades in database """ def trade_win_loss(trade): if trade.close_profit > 0: - return 'Wins' + return 'wins' elif trade.close_profit < 0: - return 'Losses' + return 'losses' else: - return 'Draws' + return 'draws' trades = trades = Trade.get_trades([Trade.is_open.is_(False)]) # Sell reason sell_reasons = {} for trade in trades: if trade.sell_reason not in sell_reasons: - sell_reasons[trade.sell_reason] = {'Wins': 0, 'Losses': 0, 'Draws': 0} + sell_reasons[trade.sell_reason] = {'wins': 0, 'losses': 0, 'draws': 0} sell_reasons[trade.sell_reason][trade_win_loss(trade)] += 1 # Duration - dur: Dict[str, List[int]] = {'Wins': [], 'Draws': [], 'Losses': []} + dur: Dict[str, List[int]] = {'wins': [], 'draws': [], 'losses': []} for trade in trades: if trade.close_date is not None and trade.open_date is not None: trade_dur = (trade.close_date - trade.open_date).total_seconds() dur[trade_win_loss(trade)].append(trade_dur) - wins_dur = sum(dur['Wins']) / len(dur['Wins']) if len(dur['Wins']) > 0 else 'N/A' - draws_dur = sum(dur['Draws']) / len(dur['Draws']) if len(dur['Draws']) > 0 else 'N/A' - losses_dur = sum(dur['Losses']) / len(dur['Losses']) if len(dur['Losses']) > 0 else 'N/A' + wins_dur = sum(dur['wins']) / len(dur['wins']) if len(dur['wins']) > 0 else 'N/A' + draws_dur = sum(dur['draws']) / len(dur['draws']) if len(dur['draws']) > 0 else 'N/A' + losses_dur = sum(dur['losses']) / len(dur['losses']) if len(dur['losses']) > 0 else 'N/A' durations = {'wins': wins_dur, 'draws': draws_dur, 'losses': losses_dur} - return sell_reasons, durations + return {'sell_reasons': sell_reasons, 'durations': durations} def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 76d9292b4..25965e05f 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -3,9 +3,9 @@ """ This module manage Telegram communication """ -from datetime import timedelta import json import logging +from datetime import timedelta from typing import Any, Callable, Dict, List, Union import arrow @@ -395,9 +395,8 @@ class Telegram(RPC): """ Handler for /stats Show stats of recent trades - :return: None """ - sell_reasons, durations = self._rpc_stats() + stats = self._rpc_stats() sell_reasons_tabulate = [] reason_map = { @@ -409,26 +408,24 @@ class Telegram(RPC): 'force_sell': 'Forcesell', 'emergency_sell': 'Emergency Sell', } - for reason, count in sell_reasons.items(): + for reason, count in stats['sell_reasons'].items(): sell_reasons_tabulate.append([ reason_map.get(reason, reason), sum(count.values()), - count['Wins'], - # count['Draws'], - count['Losses'] + count['wins'], + count['losses'] ]) sell_reasons_msg = tabulate( sell_reasons_tabulate, headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] ) - + durations = stats['durations'] duration_msg = tabulate([ ['Wins', str(timedelta(seconds=durations['wins'])) if durations['wins'] != 'N/A' else 'N/A'], - # ['Draws', str(timedelta(seconds=durations['draws']))], ['Losses', str(timedelta(seconds=durations['losses'])) if durations['losses'] != 'N/A' else 'N/A'] - ], + ], headers=['', 'Avg. Duration'] ) msg = (f"""```\n{sell_reasons_msg}```\n```\n{duration_msg}```""") From 81410fb4044c0d6238441ce86dcaed95d3d3e975 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 15:03:16 +0100 Subject: [PATCH 14/32] Document /stats for telegram --- docs/telegram-usage.md | 1 + freqtrade/rpc/telegram.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index f4bd0a12a..c940f59ac 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -113,6 +113,7 @@ official commands. You can ask at any moment for help with `/help`. | `/performance` | Show performance of each finished trade grouped by pair | `/balance` | Show account balance per currency | `/daily ` | Shows profit or loss per day, over the last n days (n defaults to 7) +| `/stats` | Shows Wins / losses by Sell reason as well as Avg. holding durations for buys and sells | `/whitelist` | Show the current whitelist | `/blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. | `/edge` | Show validated pairs by Edge if it is enabled. diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 25965e05f..b6c0a1f3f 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -787,6 +787,8 @@ class Telegram(RPC): "*/delete :* `Instantly delete the given trade in the database`\n" "*/performance:* `Show performance of each finished trade grouped by pair`\n" "*/daily :* `Shows profit or loss per day, over the last n days`\n" + "*/stats:* `Shows Wins / losses by Sell reason as well as " + "Avg. holding durationsfor buys and sells.`\n" "*/count:* `Show number of active trades compared to allowed number of trades`\n" "*/locks:* `Show currently locked pairs`\n" "*/balance:* `Show account balance per currency`\n" From 3ab5514697c294a9ab5918dc44e408f2f88bb341 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 15:07:08 +0100 Subject: [PATCH 15/32] Add API endpoint for /stats --- docs/rest-api.md | 4 ++++ freqtrade/rpc/api_server.py | 14 ++++++++++++++ scripts/rest_client.py | 7 +++++++ tests/rpc/test_rpc_apiserver.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/docs/rest-api.md b/docs/rest-api.md index 7726ab875..9bb35ce91 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -127,6 +127,7 @@ python3 scripts/rest_client.py --config rest_config.json [optional par | `performance` | Show performance of each finished trade grouped by pair. | `balance` | Show account balance per currency. | `daily ` | Shows profit or loss per day, over the last n days (n defaults to 7). +| `stats` | Display a summary of profit / loss reasons as well as average holding times. | `whitelist` | Show the current whitelist. | `blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. | `edge` | Show validated pairs by Edge if it is enabled. @@ -229,6 +230,9 @@ show_config start Start the bot if it's in the stopped state. +stats + Return the stats report (durations, sell-reasons). + status Get the status of open trades. diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 8c2c203e6..c86aa1fa7 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -198,6 +198,8 @@ class ApiServer(RPC): self.app.add_url_rule(f'{BASE_URI}/logs', 'log', view_func=self._get_logs, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/profit', 'profit', view_func=self._profit, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/stats', 'stats', + view_func=self._stats, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/performance', 'performance', view_func=self._performance, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/status', 'status', @@ -417,6 +419,18 @@ class ApiServer(RPC): return jsonify(stats) + @require_login + @rpc_catch_errors + def _stats(self): + """ + Handler for /stats. + Returns a Object with "durations" and "sell_reasons" as keys. + """ + + stats = self._rpc_stats() + + return jsonify(stats) + @require_login @rpc_catch_errors def _performance(self): diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 268e81397..2232b8421 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -139,6 +139,13 @@ class FtRestClient(): """ return self._get("profit") + def stats(self): + """Return the stats report (durations, sell-reasons). + + :return: json object + """ + return self._get("stats") + def performance(self): """Return the performance of the different coins. diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 0dc43474f..2daa32bc7 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -559,6 +559,35 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li } +@pytest.mark.usefixtures("init_persistence") +def test_api_stats(botclient, mocker, ticker, fee, markets,): + ftbot, client = botclient + patch_get_signal(ftbot, (True, False)) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value=ticker), + fetch_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + + rc = client_get(client, f"{BASE_URI}/stats") + assert_response(rc, 200) + assert 'durations' in rc.json + assert 'sell_reasons' in rc.json + + create_mock_trades(fee) + + rc = client_get(client, f"{BASE_URI}/stats") + assert_response(rc, 200) + assert 'durations' in rc.json + assert 'sell_reasons' in rc.json + + assert 'wins' in rc.json['durations'] + assert 'losses' in rc.json['durations'] + assert 'draws' in rc.json['durations'] + + def test_api_performance(botclient, mocker, ticker, fee): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) From 118a22d0104cf6f8298f0f7c72639c450f0413b0 Mon Sep 17 00:00:00 2001 From: Samaoo Date: Tue, 8 Dec 2020 18:04:26 +0100 Subject: [PATCH 16/32] Update data-download.md --- docs/data-download.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-download.md b/docs/data-download.md index e9c5c1865..2d77a8a17 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -8,7 +8,7 @@ If no additional parameter is specified, freqtrade will download data for `"1m"` Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. -You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101`). For incremental downloads, the relative approach should be used. +You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used. !!! Tip "Tip: Updating existing data" If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. From d9a86158f4ef44b5f172e4b3af7c257fd7d5c85f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Dec 2020 19:46:54 +0100 Subject: [PATCH 17/32] Add cmake to support raspberry 64bit installs --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 5cc0e03f4..f00bf0836 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -112,7 +112,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces ``` bash - sudo apt-get install python3-venv libatlas-base-dev + sudo apt-get install python3-venv libatlas-base-dev cmake # Use pywheels.org to speed up installation sudo echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > tee /etc/pip.conf From e6b3e645340da773d20f3b96144872b4bd115c63 Mon Sep 17 00:00:00 2001 From: David Martinez Martin Date: Wed, 9 Dec 2020 03:27:59 +0100 Subject: [PATCH 18/32] Update dockerfile to multistage This change reduce the image size from 727Mb to 469Mb. --- Dockerfile | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2be65274e..8840a707a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,40 @@ -FROM python:3.8.6-slim-buster +FROM python:3.8.6-slim-buster as base -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev sqlite3 \ - && apt-get clean \ - && pip install --upgrade pip +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 # Prepare environment RUN mkdir /freqtrade WORKDIR /freqtrade +# Install dependencies +FROM base as python-deps +RUN apt-get update \ + && apt-get -y install git curl build-essential libssl-dev \ + && apt-get clean \ + && pip install --upgrade pip + # Install TA-lib COPY build_helpers/* /tmp/ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* - ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies COPY requirements.txt requirements-hyperopt.txt /freqtrade/ -RUN pip install numpy --no-cache-dir \ - && pip install -r requirements-hyperopt.txt --no-cache-dir +RUN pip install --user --no-cache-dir numpy \ + && pip install --user --no-cache-dir -r requirements-hyperopt.txt + +# Copy dependencies to runtime-image +FROM base as runtime-image +COPY --from=python-deps /usr/local/lib /usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib + +COPY --from=python-deps /root/.local /root/.local +ENV PATH=/root/.local/bin:$PATH + # Install and execute COPY . /freqtrade/ From f1af2972e2b23b627a99e5a3645031e489066470 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Dec 2020 07:52:58 +0100 Subject: [PATCH 19/32] Ensure non-defined attributes fail correctly Remove unnecessary check, as stoploss cannot be none (it's mandatory and a number) --- freqtrade/freqtradebot.py | 3 +-- freqtrade/resolvers/strategy_resolver.py | 27 +++++++++++++++++------- freqtrade/strategy/interface.py | 3 +-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c8d281852..15aa3416c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -519,8 +519,7 @@ class FreqtradeBot: # reserve some percent defined in config (5% default) + stoploss amount_reserve_percent = 1.0 - self.config.get('amount_reserve_percent', constants.DEFAULT_AMOUNT_RESERVE_PERCENT) - if self.strategy.stoploss is not None: - amount_reserve_percent += self.strategy.stoploss + amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 63a3f784e..73af00fee 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -88,9 +88,6 @@ class StrategyResolver(IResolver): StrategyResolver._override_attribute_helper(strategy, config, attribute, default) - # Assign deprecated variable - to not break users code relying on this. - strategy.ticker_interval = strategy.timeframe - # Loop this list again to have output combined for attribute, _, subkey in attributes: if subkey and attribute in config[subkey]: @@ -98,11 +95,7 @@ class StrategyResolver(IResolver): elif attribute in config: logger.info("Strategy using %s: %s", attribute, config[attribute]) - # Sort and apply type conversions - strategy.minimal_roi = OrderedDict(sorted( - {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), - key=lambda t: t[0])) - strategy.stoploss = float(strategy.stoploss) + StrategyResolver._normalize_attributes(strategy) StrategyResolver._strategy_sanity_validations(strategy) return strategy @@ -131,6 +124,24 @@ class StrategyResolver(IResolver): setattr(strategy, attribute, default) config[attribute] = default + @staticmethod + def _normalize_attributes(strategy: IStrategy) -> IStrategy: + """ + Normalize attributes to have the correct type. + """ + # Assign deprecated variable - to not break users code relying on this. + if hasattr(strategy, 'timeframe'): + strategy.ticker_interval = strategy.timeframe + + # Sort and apply type conversions + if hasattr(strategy, 'minimal_roi'): + strategy.minimal_roi = OrderedDict(sorted( + {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), + key=lambda t: t[0])) + if hasattr(strategy, 'stoploss'): + strategy.stoploss = float(strategy.stoploss) + return strategy + @staticmethod def _strategy_sanity_validations(strategy): if not all(k in strategy.order_types for k in REQUIRED_ORDERTYPES): diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 81f4e7651..125211a85 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -551,8 +551,7 @@ class IStrategy(ABC): # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. - if ((self.stoploss is not None) and - (trade.stop_loss >= current_rate) and + if ((trade.stop_loss >= current_rate) and (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS From 57080982566e295747da2286d87f94f012e812cc Mon Sep 17 00:00:00 2001 From: David Martinez Martin Date: Wed, 9 Dec 2020 10:34:38 +0100 Subject: [PATCH 20/32] Move ENV PATH to base image --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8840a707a..f85dfb0c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONFAULTHANDLER 1 +ENV PATH=/root/.local/bin:$PATH # Prepare environment RUN mkdir /freqtrade @@ -13,7 +14,7 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps RUN apt-get update \ - && apt-get -y install git curl build-essential libssl-dev \ + && apt-get -y install curl build-essential libssl-dev \ && apt-get clean \ && pip install --upgrade pip @@ -33,7 +34,7 @@ COPY --from=python-deps /usr/local/lib /usr/local/lib ENV LD_LIBRARY_PATH /usr/local/lib COPY --from=python-deps /root/.local /root/.local -ENV PATH=/root/.local/bin:$PATH + # Install and execute From 25f8e0cc57b050c10ecab3cbf1d2b712008fd341 Mon Sep 17 00:00:00 2001 From: David Martinez Martin Date: Wed, 9 Dec 2020 11:28:45 +0100 Subject: [PATCH 21/32] Added git packages for future dependencies --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f85dfb0c7..602e6a28c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev \ + && apt-get -y install curl build-essential libssl-dev git \ && apt-get clean \ && pip install --upgrade pip From f5817063b75b5957d29783c3f5f1b9eda19d20fc Mon Sep 17 00:00:00 2001 From: Samaoo Date: Wed, 9 Dec 2020 15:53:38 +0100 Subject: [PATCH 22/32] Update backtesting.md --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 3058d1b57..1fc9f3d73 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -285,7 +285,7 @@ Since backtesting lacks some detailed information about what happens within a ca - sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) - sells are never "below the candle", so a ROI of 2% may result in a sell at 2.4% if low was at 2.4% profit - Forcesells caused by `=-1` ROI entries use low as sell value, unless N falls on the candle open (e.g. `120: -1` for 1h candles) -- Stoploss sells happen exactly at stoploss price, even if low was lower +- Stoploss sells happen exactly at stoploss price, even if low was lower, but the loss will be 0.32% lower than the stoploss price. - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes - Low happens before high for stoploss, protecting capital first - Trailing stoploss From af53dfbfab71e61cc3c50200ed2bb26c283720b7 Mon Sep 17 00:00:00 2001 From: Samaoo Date: Wed, 9 Dec 2020 15:57:15 +0100 Subject: [PATCH 23/32] Update backtesting.md --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 1fc9f3d73..de54c4c91 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -285,7 +285,7 @@ Since backtesting lacks some detailed information about what happens within a ca - sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) - sells are never "below the candle", so a ROI of 2% may result in a sell at 2.4% if low was at 2.4% profit - Forcesells caused by `=-1` ROI entries use low as sell value, unless N falls on the candle open (e.g. `120: -1` for 1h candles) -- Stoploss sells happen exactly at stoploss price, even if low was lower, but the loss will be 0.32% lower than the stoploss price. +- Stoploss sells happen exactly at stoploss price, even if low was lower, but the loss will be 0.32% lower than the stoploss price - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes - Low happens before high for stoploss, protecting capital first - Trailing stoploss From 33f330256b02098458cbd18e28d970a735efb5e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Dec 2020 20:26:11 +0100 Subject: [PATCH 24/32] Reorder commands on telegram init --- freqtrade/rpc/telegram.py | 2 +- tests/conftest_trades.py | 10 ++++++++-- tests/rpc/test_rpc_telegram.py | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index b6c0a1f3f..fa36cfee9 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -99,6 +99,7 @@ class Telegram(RPC): CommandHandler('trades', self._trades), CommandHandler('delete', self._delete_trade), CommandHandler('performance', self._performance), + CommandHandler('stats', self._stats), CommandHandler('daily', self._daily), CommandHandler('count', self._count), CommandHandler('locks', self._locks), @@ -111,7 +112,6 @@ class Telegram(RPC): CommandHandler('edge', self._edge), CommandHandler('help', self._help), CommandHandler('version', self._version), - CommandHandler('stats', self._stats), ] for handle in handles: self._updater.dispatcher.add_handler(handle) diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index fac822b2b..e84722041 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta, timezone + from freqtrade.persistence.models import Order, Trade @@ -82,7 +84,9 @@ def mock_trade_2(fee): is_open=False, open_order_id='dry_run_sell_12345', strategy='DefaultStrategy', - sell_reason='sell_signal' + sell_reason='sell_signal', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc), ) o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy') trade.orders.append(o) @@ -135,7 +139,9 @@ def mock_trade_3(fee): close_profit=0.01, exchange='bittrex', is_open=False, - sell_reason='roi' + sell_reason='roi', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc), ) o = Order.parse_from_ccxt_object(mock_order_3(), 'XRP/BTC', 'buy') trade.orders.append(o) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 725c1411e..ecad05683 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -74,10 +74,10 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: message_str = ("rpc.telegram is listening for following commands: [['status'], ['profit'], " "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], ['trades'], " - "['delete'], ['performance'], ['daily'], ['count'], ['locks'], " + "['delete'], ['performance'], ['stats'], ['daily'], ['count'], ['locks'], " "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], ['stopbuy'], " - "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version'], " - "['stats']]") + "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version']" + "]") assert log_has(message_str, caplog) From ca99d484fcd852561f8acbb3bd9cbb879ddc724d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Dec 2020 07:39:50 +0100 Subject: [PATCH 25/32] Refactor to use list comprehension --- freqtrade/rpc/telegram.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index fa36cfee9..c54000677 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -398,7 +398,6 @@ class Telegram(RPC): """ stats = self._rpc_stats() - sell_reasons_tabulate = [] reason_map = { 'roi': 'ROI', 'stop_loss': 'Stoploss', @@ -408,13 +407,14 @@ class Telegram(RPC): 'force_sell': 'Forcesell', 'emergency_sell': 'Emergency Sell', } - for reason, count in stats['sell_reasons'].items(): - sell_reasons_tabulate.append([ + sell_reasons_tabulate = [ + [ reason_map.get(reason, reason), sum(count.values()), count['wins'], count['losses'] - ]) + ] for reason, count in stats['sell_reasons'].items() + ] sell_reasons_msg = tabulate( sell_reasons_tabulate, headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] From 201cc67e0503c98ad962ea1a0a53b4763fdedd81 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Dec 2020 19:21:20 +0100 Subject: [PATCH 26/32] Rename open_trade_price to "open_trade_value" --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence/models.py | 22 +++++++++++----------- tests/rpc/test_rpc.py | 4 ++-- tests/rpc/test_rpc_apiserver.py | 4 ++-- tests/test_persistence.py | 28 ++++++++++++++-------------- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c8d281852..59da58e1b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1396,7 +1396,7 @@ class FreqtradeBot: abs_tol=constants.MATH_CLOSE_PREC): order['amount'] = new_amount order.pop('filled', None) - trade.recalc_open_trade_price() + trade.recalc_open_trade_value() except DependencyException as exception: logger.warning("Could not update trade amount: %s", exception) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 6027908da..67871f96b 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -217,7 +217,7 @@ class Trade(_DECL_BASE): fee_close_currency = Column(String, nullable=True) open_rate = Column(Float) open_rate_requested = Column(Float) - # open_trade_price - calculated via _calc_open_trade_price + # open_trade_price - calculated via _calc_open_trade_value open_trade_price = Column(Float) close_rate = Column(Float) close_rate_requested = Column(Float) @@ -252,7 +252,7 @@ class Trade(_DECL_BASE): def __init__(self, **kwargs): super().__init__(**kwargs) - self.recalc_open_trade_price() + self.recalc_open_trade_value() def __repr__(self): open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' @@ -284,7 +284,7 @@ class Trade(_DECL_BASE): 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), 'open_rate': self.open_rate, 'open_rate_requested': self.open_rate_requested, - 'open_trade_price': round(self.open_trade_price, 8), + 'open_trade_value': round(self.open_trade_price, 8), 'close_date_hum': (arrow.get(self.close_date).humanize() if self.close_date else None), @@ -389,7 +389,7 @@ class Trade(_DECL_BASE): # Update open rate and actual amount self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price')) self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount')) - self.recalc_open_trade_price() + self.recalc_open_trade_value() if self.is_open: logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.') self.open_order_id = None @@ -464,7 +464,7 @@ class Trade(_DECL_BASE): Trade.session.delete(self) Trade.session.flush() - def _calc_open_trade_price(self) -> float: + def _calc_open_trade_value(self) -> float: """ Calculate the open_rate including open_fee. :return: Price in of the open trade incl. Fees @@ -473,14 +473,14 @@ class Trade(_DECL_BASE): fees = buy_trade * Decimal(self.fee_open) return float(buy_trade + fees) - def recalc_open_trade_price(self) -> None: + def recalc_open_trade_value(self) -> None: """ - Recalculate open_trade_price. + Recalculate open_trade_value. Must be called whenever open_rate or fee_open is changed. """ - self.open_trade_price = self._calc_open_trade_price() + self.open_trade_price = self._calc_open_trade_value() - def calc_close_trade_price(self, rate: Optional[float] = None, + def calc_close_trade_value(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ Calculate the close_rate including fee @@ -507,7 +507,7 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: profit in stake currency as float """ - close_trade_price = self.calc_close_trade_price( + close_trade_price = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) @@ -523,7 +523,7 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit ratio as float """ - close_trade_price = self.calc_close_trade_price( + close_trade_price = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 47e0f763d..4b36f4b4e 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -62,7 +62,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'fee_close_cost': ANY, 'fee_close_currency': ANY, 'open_rate_requested': ANY, - 'open_trade_price': 0.0010025, + 'open_trade_value': 0.0010025, 'close_rate_requested': ANY, 'sell_reason': ANY, 'sell_order_status': ANY, @@ -127,7 +127,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'fee_close_cost': ANY, 'fee_close_currency': ANY, 'open_rate_requested': ANY, - 'open_trade_price': ANY, + 'open_trade_value': ANY, 'close_rate_requested': ANY, 'sell_reason': ANY, 'sell_order_status': ANY, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 0dc43474f..8e5a66998 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -678,7 +678,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'min_rate': 1.098e-05, 'open_order_id': None, 'open_rate_requested': 1.098e-05, - 'open_trade_price': 0.0010025, + 'open_trade_value': 0.0010025, 'sell_reason': None, 'sell_order_status': None, 'strategy': 'DefaultStrategy', @@ -805,7 +805,7 @@ def test_api_forcebuy(botclient, mocker, fee): 'min_rate': None, 'open_order_id': '123456', 'open_rate_requested': None, - 'open_trade_price': 0.24605460, + 'open_trade_value': 0.24605460, 'sell_reason': None, 'sell_order_status': None, 'strategy': None, diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 41b99b34f..a7ac8ed94 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -177,10 +177,10 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): trade.open_order_id = 'something' trade.update(limit_buy_order) - assert trade._calc_open_trade_price() == 0.0010024999999225068 + assert trade._calc_open_trade_value() == 0.0010024999999225068 trade.update(limit_sell_order) - assert trade.calc_close_trade_price() == 0.0010646656050132426 + assert trade.calc_close_trade_value() == 0.0010646656050132426 # Profit in BTC assert trade.calc_profit() == 0.00006217 @@ -233,7 +233,7 @@ def test_calc_close_trade_price_exception(limit_buy_order, fee): trade.open_order_id = 'something' trade.update(limit_buy_order) - assert trade.calc_close_trade_price() == 0.0 + assert trade.calc_close_trade_value() == 0.0 @pytest.mark.usefixtures("init_persistence") @@ -277,7 +277,7 @@ def test_update_invalid_order(limit_buy_order): @pytest.mark.usefixtures("init_persistence") -def test_calc_open_trade_price(limit_buy_order, fee): +def test_calc_open_trade_value(limit_buy_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, @@ -291,10 +291,10 @@ def test_calc_open_trade_price(limit_buy_order, fee): trade.update(limit_buy_order) # Buy @ 0.00001099 # Get the open rate price with the standard fee rate - assert trade._calc_open_trade_price() == 0.0010024999999225068 + assert trade._calc_open_trade_value() == 0.0010024999999225068 trade.fee_open = 0.003 # Get the open rate price with a custom fee rate - assert trade._calc_open_trade_price() == 0.001002999999922468 + assert trade._calc_open_trade_value() == 0.001002999999922468 @pytest.mark.usefixtures("init_persistence") @@ -312,14 +312,14 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order, fee): trade.update(limit_buy_order) # Buy @ 0.00001099 # Get the close rate price with a custom close rate and a regular fee rate - assert trade.calc_close_trade_price(rate=0.00001234) == 0.0011200318470471794 + assert trade.calc_close_trade_value(rate=0.00001234) == 0.0011200318470471794 # Get the close rate price with a custom close rate and a custom fee rate - assert trade.calc_close_trade_price(rate=0.00001234, fee=0.003) == 0.0011194704275749754 + assert trade.calc_close_trade_value(rate=0.00001234, fee=0.003) == 0.0011194704275749754 # Test when we apply a Sell order, and ask price with a custom fee rate trade.update(limit_sell_order) - assert trade.calc_close_trade_price(fee=0.005) == 0.0010619972701635854 + assert trade.calc_close_trade_value(fee=0.005) == 0.0010619972701635854 @pytest.mark.usefixtures("init_persistence") @@ -499,7 +499,7 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_price == trade._calc_open_trade_price() + assert trade.open_trade_price == trade._calc_open_trade_value() assert trade.close_profit_abs is None assert trade.fee_open_cost is None assert trade.fee_open_currency is None @@ -607,7 +607,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak2", caplog) assert log_has("Running database migration for trades - backup: trades_bak2", caplog) - assert trade.open_trade_price == trade._calc_open_trade_price() + assert trade.open_trade_price == trade._calc_open_trade_value() assert trade.close_profit_abs is None assert log_has("Moving open orders to Orders table.", caplog) @@ -677,7 +677,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_price == trade._calc_open_trade_price() + assert trade.open_trade_price == trade._calc_open_trade_value() assert log_has("trying trades_bak0", caplog) assert log_has("Running database migration for trades - backup: trades_bak0", caplog) @@ -803,7 +803,7 @@ def test_to_json(default_conf, fee): 'close_timestamp': None, 'open_rate': 0.123, 'open_rate_requested': None, - 'open_trade_price': 15.1668225, + 'open_trade_value': 15.1668225, 'fee_close': 0.0025, 'fee_close_cost': None, 'fee_close_currency': None, @@ -896,7 +896,7 @@ def test_to_json(default_conf, fee): 'min_rate': None, 'open_order_id': None, 'open_rate_requested': None, - 'open_trade_price': 12.33075, + 'open_trade_value': 12.33075, 'sell_reason': None, 'sell_order_status': None, 'strategy': None, From 95fd3824daa02d6889b7eaf636a51397d37f0f59 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Dec 2020 19:36:52 +0100 Subject: [PATCH 27/32] Finish renamal of open_trade_price to open_value --- freqtrade/persistence/migrations.py | 10 +++++----- freqtrade/persistence/models.py | 16 ++++++++-------- tests/test_persistence.py | 6 +++--- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 84f3ed7e6..ed976c2a9 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -53,11 +53,11 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col else: timeframe = get_column_def(cols, 'timeframe', 'null') - open_trade_price = get_column_def(cols, 'open_trade_price', + open_trade_value = get_column_def(cols, 'open_trade_value', f'amount * open_rate * (1 + {fee_open})') close_profit_abs = get_column_def( cols, 'close_profit_abs', - f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_value}") sell_order_status = get_column_def(cols, 'sell_order_status', 'null') amount_requested = get_column_def(cols, 'amount_requested', 'amount') @@ -79,7 +79,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, stoploss_order_id, stoploss_last_update, max_rate, min_rate, sell_reason, sell_order_status, strategy, - timeframe, open_trade_price, close_profit_abs + timeframe, open_trade_value, close_profit_abs ) select id, lower(exchange), case @@ -102,7 +102,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, {sell_order_status} sell_order_status, {strategy} strategy, {timeframe} timeframe, - {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs + {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs from {table_back_name} """) @@ -134,7 +134,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: table_back_name = get_backup_name(tabs, 'trades_bak') # Check for latest column - if not has_column(cols, 'amount_requested'): + if not has_column(cols, 'open_trade_value'): logger.info(f'Running database migration for trades - backup: {table_back_name}') migrate_trades_table(decl_base, inspector, engine, table_back_name, cols) # Reread columns - the above recreated the table! diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 67871f96b..06dd785e8 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -217,8 +217,8 @@ class Trade(_DECL_BASE): fee_close_currency = Column(String, nullable=True) open_rate = Column(Float) open_rate_requested = Column(Float) - # open_trade_price - calculated via _calc_open_trade_value - open_trade_price = Column(Float) + # open_trade_value - calculated via _calc_open_trade_value + open_trade_value = Column(Float) close_rate = Column(Float) close_rate_requested = Column(Float) close_profit = Column(Float) @@ -284,7 +284,7 @@ class Trade(_DECL_BASE): 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), 'open_rate': self.open_rate, 'open_rate_requested': self.open_rate_requested, - 'open_trade_value': round(self.open_trade_price, 8), + 'open_trade_value': round(self.open_trade_value, 8), 'close_date_hum': (arrow.get(self.close_date).humanize() if self.close_date else None), @@ -478,7 +478,7 @@ class Trade(_DECL_BASE): Recalculate open_trade_value. Must be called whenever open_rate or fee_open is changed. """ - self.open_trade_price = self._calc_open_trade_value() + self.open_trade_value = self._calc_open_trade_value() def calc_close_trade_value(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: @@ -507,11 +507,11 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: profit in stake currency as float """ - close_trade_price = self.calc_close_trade_value( + close_trade_value = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit = close_trade_price - self.open_trade_price + profit = close_trade_value - self.open_trade_value return float(f"{profit:.8f}") def calc_profit_ratio(self, rate: Optional[float] = None, @@ -523,11 +523,11 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit ratio as float """ - close_trade_price = self.calc_close_trade_value( + close_trade_value = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit_ratio = (close_trade_price / self.open_trade_price) - 1 + profit_ratio = (close_trade_value / self.open_trade_value) - 1 return float(f"{profit_ratio:.8f}") def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index a7ac8ed94..7487b2ef5 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -499,7 +499,7 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_price == trade._calc_open_trade_value() + assert trade.open_trade_value == trade._calc_open_trade_value() assert trade.close_profit_abs is None assert trade.fee_open_cost is None assert trade.fee_open_currency is None @@ -607,7 +607,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak2", caplog) assert log_has("Running database migration for trades - backup: trades_bak2", caplog) - assert trade.open_trade_price == trade._calc_open_trade_value() + assert trade.open_trade_value == trade._calc_open_trade_value() assert trade.close_profit_abs is None assert log_has("Moving open orders to Orders table.", caplog) @@ -677,7 +677,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_price == trade._calc_open_trade_value() + assert trade.open_trade_value == trade._calc_open_trade_value() assert log_has("trying trades_bak0", caplog) assert log_has("Running database migration for trades - backup: trades_bak0", caplog) From 6107878f4e573b6f0b00eb56f9d0704408a2a806 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 12 Dec 2020 07:08:29 +0100 Subject: [PATCH 28/32] Bump ccxt to 1.39.10 closes #4051 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 105839f0d..e0c5ac072 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.19.4 pandas==1.1.4 -ccxt==1.38.87 +ccxt==1.39.10 aiohttp==3.7.3 SQLAlchemy==1.3.20 python-telegram-bot==13.1 From b45c2fb1d015847ba0b3fbe1c8059d7267e99d6a Mon Sep 17 00:00:00 2001 From: Samaoo Date: Sat, 12 Dec 2020 10:27:17 +0100 Subject: [PATCH 29/32] Update backtesting.md --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index de54c4c91..27bfebe37 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -285,7 +285,7 @@ Since backtesting lacks some detailed information about what happens within a ca - sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) - sells are never "below the candle", so a ROI of 2% may result in a sell at 2.4% if low was at 2.4% profit - Forcesells caused by `=-1` ROI entries use low as sell value, unless N falls on the candle open (e.g. `120: -1` for 1h candles) -- Stoploss sells happen exactly at stoploss price, even if low was lower, but the loss will be 0.32% lower than the stoploss price +- Stoploss sells happen exactly at stoploss price, even if low was lower, but the loss will be `2 * fees` higher than the stoploss price - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes - Low happens before high for stoploss, protecting capital first - Trailing stoploss From 181b88dc753354b34ca65162541823e1bf7b5258 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 12 Dec 2020 10:52:27 +0100 Subject: [PATCH 30/32] Don't accept too high fees, assuming they are erroneous Forces fallback to "detection from trades" --- freqtrade/freqtradebot.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ada9889a6..c86fb616b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1448,13 +1448,16 @@ class FreqtradeBot: fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) logger.info(f"Fee for Trade {trade} [{order.get('side')}]: " f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}") - - trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) - if trade_base_currency == fee_currency: - # Apply fee to amount - return self.apply_fee_conditional(trade, trade_base_currency, - amount=order_amount, fee_abs=fee_cost) - return order_amount + if fee_rate is None or fee_rate < 0.02: + # Reject all fees that report as > 2%. + # These are most likely caused by a parsing bug in ccxt + # due to multiple trades (https://github.com/ccxt/ccxt/issues/8025) + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + if trade_base_currency == fee_currency: + # Apply fee to amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=order_amount, fee_abs=fee_cost) + return order_amount return self.fee_detection_from_trades(trade, order, order_amount) def fee_detection_from_trades(self, trade: Trade, order: Dict, order_amount: float) -> float: From 3ee7fe64ba3e957ff5110f07b06c3f7265060a1e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 12 Dec 2020 11:25:56 +0100 Subject: [PATCH 31/32] Clean up some tests --- tests/conftest.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 079a521ed..e2e4788b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1588,16 +1588,7 @@ def fetch_trades_result(): @pytest.fixture(scope="function") def trades_for_order2(): - return [{'info': {'id': 34567, - 'orderId': 123456, - 'price': '0.24544100', - 'qty': '8.00000000', - 'commission': '0.00800000', - 'commissionAsset': 'LTC', - 'time': 1521663363189, - 'isBuyer': True, - 'isMaker': False, - 'isBestMatch': True}, + return [{'info': {}, 'timestamp': 1521663363189, 'datetime': '2018-03-21T20:16:03.189Z', 'symbol': 'LTC/ETH', @@ -1609,16 +1600,7 @@ def trades_for_order2(): 'cost': 1.963528, 'amount': 4.0, 'fee': {'cost': 0.004, 'currency': 'LTC'}}, - {'info': {'id': 34567, - 'orderId': 123456, - 'price': '0.24544100', - 'qty': '8.00000000', - 'commission': '0.00800000', - 'commissionAsset': 'LTC', - 'time': 1521663363189, - 'isBuyer': True, - 'isMaker': False, - 'isBestMatch': True}, + {'info': {}, 'timestamp': 1521663363189, 'datetime': '2018-03-21T20:16:03.189Z', 'symbol': 'LTC/ETH', From 14647fb5f08254d4b85c528f59db66c697c43d83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 12 Dec 2020 11:43:47 +0100 Subject: [PATCH 32/32] Add tests for update fee --- tests/conftest.py | 8 +++++++ tests/test_freqtradebot.py | 44 +++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index e2e4788b6..5d358f015 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1614,6 +1614,14 @@ def trades_for_order2(): 'fee': {'cost': 0.004, 'currency': 'LTC'}}] +@pytest.fixture(scope="function") +def trades_for_order3(trades_for_order2): + # Different fee currencies for each trade + trades_for_order = deepcopy(trades_for_order2) + trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'} + return trades_for_order + + @pytest.fixture def buy_order_fee(): return { diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 6adef510f..459a09c0c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3718,6 +3718,48 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) + assert trade.fee_open == 0.001 + assert trade.fee_close == 0.001 + assert trade.fee_open_cost is not None + assert trade.fee_open_currency is not None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None + + +def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, + mocker, markets): + # Different fee currency on both trades + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3) + amount = float(sum(x['amount'] for x in trades_for_order3)) + default_conf['stake_currency'] = 'ETH' + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.245441, + open_order_id="123456" + ) + # Fake markets entry to enable fee parsing + markets['BNB/ETH'] = markets['ETH/BTC'] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': 0.19, 'last': 0.2}) + + # Amount is reduced by "fee" + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005) + assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', + caplog) + # Overall fee is average of both trade's fee + assert trade.fee_open == 0.001518575 + assert trade.fee_open_cost is not None + assert trade.fee_open_currency is not None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None + def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): @@ -4264,7 +4306,7 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): freqtrade = get_patched_freqtradebot(mocker, default_conf) def patch_with_fee(order): - order.update({'fee': {'cost': 0.1, 'rate': 0.2, + order.update({'fee': {'cost': 0.1, 'rate': 0.01, 'currency': order['symbol'].split('/')[0]}}) return order