From 7d77aff2894d864f3f32417434a58606da2b05da Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Nov 2021 09:24:20 +0100 Subject: [PATCH 01/55] Add some compatibility around buy_tag --- docs/strategy-advanced.md | 6 +++--- docs/strategy-customization.md | 2 +- docs/webhook-config.md | 6 +++--- freqtrade/freqtradebot.py | 5 +++++ freqtrade/persistence/models.py | 1 + freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/telegram.py | 8 ++++---- tests/test_persistence.py | 2 ++ 8 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 573d184ff..908165fc7 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -77,7 +77,7 @@ class AwesomeStrategy(IStrategy): *** -## Buy Tag +## Enter Tag When your strategy has multiple buy signals, you can name the signal that triggered. Then you can access you buy signal on `custom_sell` @@ -89,7 +89,7 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: (dataframe['rsi'] < 35) & (dataframe['volume'] > 0) ), - ['buy', 'buy_tag']] = (1, 'buy_signal_rsi') + ['buy', 'enter_tag']] = (1, 'buy_signal_rsi') return dataframe @@ -104,7 +104,7 @@ def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_r ``` !!! Note - `buy_tag` is limited to 100 characters, remaining data will be truncated. + `enter_tag` is limited to 100 characters, remaining data will be truncated. ## Exit tag diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 178ed108b..e90d87c4a 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -498,7 +498,7 @@ for more information. & (dataframe['volume'] > 0) ), - ['buy', 'buy_tag']] = (1, 'buy_signal_rsi') + ['buy', 'enter_tag']] = (1, 'buy_signal_rsi') return dataframe ``` diff --git a/docs/webhook-config.md b/docs/webhook-config.md index ec944cb50..43aa0502c 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -83,7 +83,7 @@ Possible parameters are: * `fiat_currency` * `order_type` * `current_rate` -* `buy_tag` +* `enter_tag` ### Webhookbuycancel @@ -101,7 +101,7 @@ Possible parameters are: * `fiat_currency` * `order_type` * `current_rate` -* `buy_tag` +* `enter_tag` ### Webhookbuyfill @@ -117,7 +117,7 @@ Possible parameters are: * `stake_amount` * `stake_currency` * `fiat_currency` -* `buy_tag` +* `enter_tag` ### Webhooksell diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d234ebb07..237c07060 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -755,6 +755,7 @@ class FreqtradeBot(LoggingMixin): 'trade_id': trade.id, 'type': RPCMessageType.SHORT if trade.is_short else RPCMessageType.BUY, 'buy_tag': trade.buy_tag, + 'enter_tag': trade.buy_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'limit': trade.open_rate, @@ -780,6 +781,7 @@ class FreqtradeBot(LoggingMixin): 'trade_id': trade.id, 'type': msg_type, 'buy_tag': trade.buy_tag, + 'enter_tag': trade.buy_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'limit': trade.open_rate, @@ -802,6 +804,7 @@ class FreqtradeBot(LoggingMixin): 'trade_id': trade.id, 'type': msg_type, 'buy_tag': trade.buy_tag, + 'enter_tag': trade.buy_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'open_rate': trade.open_rate, @@ -1384,6 +1387,7 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, + 'enter_tag': trade.buy_tag, 'buy_tag': trade.buy_tag, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, @@ -1428,6 +1432,7 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, + 'enter_tag': trade.buy_tag, 'buy_tag': trade.buy_tag, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f9df45111..f73edbe02 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -390,6 +390,7 @@ class LocalTrade(): 'stake_amount': round(self.stake_amount, 8), 'strategy': self.strategy, 'buy_tag': self.buy_tag, + 'enter_tag': self.buy_tag, 'timeframe': self.timeframe, 'fee_open': self.fee_open, diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index d6a861011..db4378a51 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -185,6 +185,7 @@ class TradeSchema(BaseModel): stake_amount: float strategy: str buy_tag: Optional[str] + enter_tag: Optional[str] timeframe: int fee_open: Optional[float] fee_open_cost: Optional[float] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0e1a6fe27..3bcc6adf2 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -226,7 +226,7 @@ class Telegram(RPCHandler): f"{emoji} *{msg['exchange']}:* {'Bought' if is_fill else 'Buying'} {msg['pair']}" f" (#{msg['trade_id']})\n" ) - message += f"*Buy Tag:* `{msg['buy_tag']}`\n" if msg.get('buy_tag', None) else "" + message += f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag', None) else "" message += f"*Amount:* `{msg['amount']:.8f}`\n" if msg['type'] == RPCMessageType.BUY_FILL: @@ -251,7 +251,7 @@ class Telegram(RPCHandler): microsecond=0) - msg['open_date'].replace(microsecond=0) msg['duration_min'] = msg['duration'].total_seconds() / 60 - msg['buy_tag'] = msg['buy_tag'] if "buy_tag" in msg.keys() else None + msg['enter_tag'] = msg['enter_tag'] if "enter_tag" in msg.keys() else None msg['emoji'] = self._get_sell_emoji(msg) # Check if all sell properties are available. @@ -397,7 +397,7 @@ class Telegram(RPCHandler): "*Trade ID:* `{trade_id}` `(since {open_date_hum})`", "*Current Pair:* {pair}", "*Amount:* `{amount} ({stake_amount} {base_currency})`", - "*Buy Tag:* `{buy_tag}`" if r['buy_tag'] else "", + "*Enter Tag:* `{enter_tag}`" if r['enter_tag'] else "", "*Open Rate:* `{open_rate:.8f}`", "*Close Rate:* `{close_rate}`" if r['close_rate'] else "", "*Current Rate:* `{current_rate:.8f}`", @@ -989,7 +989,7 @@ class Telegram(RPCHandler): output = "Buy Tag Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i+1}.\t {trade['buy_tag']}\t" + f"{i+1}.\t {trade['enter_tag']}\t" f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 2f5f61a15..9df4f511a 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1602,6 +1602,7 @@ def test_to_json(default_conf, fee): 'max_rate': None, 'strategy': None, 'buy_tag': None, + 'enter_tag': None, 'timeframe': None, 'exchange': 'binance', 'leverage': None, @@ -1675,6 +1676,7 @@ def test_to_json(default_conf, fee): 'sell_order_status': None, 'strategy': None, 'buy_tag': 'buys_signal_001', + 'enter_tag': 'buys_signal_001', 'timeframe': None, 'exchange': 'binance', 'leverage': None, From 36deced00bbef878e5858fdfd7da26d2dbdd1fd5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Nov 2021 09:51:16 +0100 Subject: [PATCH 02/55] Remove more buy_tag references --- docs/deprecated.md | 21 ++++++++++++++ docs/strategy-advanced.md | 2 +- freqtrade/data/btanalysis.py | 2 +- freqtrade/freqtradebot.py | 23 ++++++++-------- freqtrade/optimize/backtesting.py | 2 +- freqtrade/persistence/migrations.py | 6 ++-- freqtrade/persistence/models.py | 38 ++++++++++++++++---------- freqtrade/rpc/rpc.py | 4 +-- freqtrade/rpc/telegram.py | 14 ++++++---- tests/conftest.py | 2 -- tests/conftest_trades.py | 6 ++-- tests/optimize/test_backtest_detail.py | 2 +- tests/optimize/test_backtesting.py | 2 +- tests/rpc/test_rpc.py | 30 ++++++++++---------- tests/rpc/test_rpc_apiserver.py | 2 ++ tests/rpc/test_rpc_telegram.py | 26 ++++++++++-------- tests/test_freqtradebot.py | 5 ++++ tests/test_persistence.py | 6 ++-- 18 files changed, 117 insertions(+), 76 deletions(-) diff --git a/docs/deprecated.md b/docs/deprecated.md index d86a7ac7a..be1d51837 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -43,3 +43,24 @@ As this does however increase risk and provides no benefit, it's been removed fo Using separate hyperopt files was deprecated in 2021.4 and was removed in 2021.9. Please switch to the new [Parametrized Strategies](hyperopt.md) to benefit from the new hyperopt interface. + +## Margin / short changes + +// TODO-lev: update version here + +## Strategy changes + +As strategies now have to support multiple different signal types, some things had to change. + +Columns: + +* `buy` -> `enter_long` +* `sell` -> `exit_long` +* `buy_tag` -> `enter_tag` + +New columns are `enter_short` and `exit_short`, which will initiate short trades (requires additional configuration!) + +### webhooks - `buy_tag` has been renamed to `enter_tag` + +This should apply only to your strategy and potentially to webhooks. +We will keep a compatibility layer for 1-2 versions (so both `buy_tag` and `enter_tag` will still work), but support for this in webhooks will disappear after that. diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 908165fc7..560b4dcb6 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -97,7 +97,7 @@ def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_r current_profit: float, **kwargs): dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) last_candle = dataframe.iloc[-1].squeeze() - if trade.buy_tag == 'buy_signal_rsi' and last_candle['rsi'] > 80: + if trade.enter_tag == 'buy_signal_rsi' and last_candle['rsi'] > 80: return 'sell_signal_rsi' return None diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index e8d878838..48b58f193 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -30,7 +30,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'fee_open', 'fee_close', 'trade_duration', 'profit_ratio', 'profit_abs', 'sell_reason', 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', - 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'buy_tag'] + 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag'] # TODO-lev: usage of the above might need compatibility code (buy_tag, is_short?, ...?) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 237c07060..bcef984dd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -721,8 +721,7 @@ class FreqtradeBot(LoggingMixin): exchange=self.exchange.id, open_order_id=order_id, strategy=self.strategy.get_strategy_name(), - # TODO-lev: compatibility layer for buy_tag (!) - buy_tag=enter_tag, + enter_tag=enter_tag, timeframe=timeframe_to_minutes(self.config['timeframe']), leverage=leverage, is_short=is_short, @@ -754,8 +753,8 @@ class FreqtradeBot(LoggingMixin): msg = { 'trade_id': trade.id, 'type': RPCMessageType.SHORT if trade.is_short else RPCMessageType.BUY, - 'buy_tag': trade.buy_tag, - 'enter_tag': trade.buy_tag, + 'buy_tag': trade.enter_tag, + 'enter_tag': trade.enter_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'limit': trade.open_rate, @@ -780,8 +779,8 @@ class FreqtradeBot(LoggingMixin): msg = { 'trade_id': trade.id, 'type': msg_type, - 'buy_tag': trade.buy_tag, - 'enter_tag': trade.buy_tag, + 'buy_tag': trade.enter_tag, + 'enter_tag': trade.enter_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'limit': trade.open_rate, @@ -803,8 +802,8 @@ class FreqtradeBot(LoggingMixin): msg = { 'trade_id': trade.id, 'type': msg_type, - 'buy_tag': trade.buy_tag, - 'enter_tag': trade.buy_tag, + 'buy_tag': trade.enter_tag, + 'enter_tag': trade.enter_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'open_rate': trade.open_rate, @@ -1387,8 +1386,8 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, - 'enter_tag': trade.buy_tag, - 'buy_tag': trade.buy_tag, + 'buy_tag': trade.enter_tag, + 'enter_tag': trade.enter_tag, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.utcnow(), @@ -1432,8 +1431,8 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, - 'enter_tag': trade.buy_tag, - 'buy_tag': trade.buy_tag, + 'buy_tag': trade.enter_tag, + 'enter_tag': trade.enter_tag, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.now(timezone.utc), diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0f4d17fd8..a18d14cbe 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -478,7 +478,7 @@ class Backtesting: fee_open=self.fee, fee_close=self.fee, is_open=True, - buy_tag=row[ENTER_TAG_IDX] if has_enter_tag else None, + enter_tag=row[ENTER_TAG_IDX] if has_enter_tag else None, exchange=self._exchange_name, is_short=(direction == 'short'), ) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 2b1d10bc1..212499df3 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -47,7 +47,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col min_rate = get_column_def(cols, 'min_rate', 'null') sell_reason = get_column_def(cols, 'sell_reason', 'null') strategy = get_column_def(cols, 'strategy', 'null') - buy_tag = get_column_def(cols, 'buy_tag', 'null') + enter_tag = get_column_def(cols, 'buy_tag', get_column_def(cols, 'enter_tag', 'null')) trading_mode = get_column_def(cols, 'trading_mode', 'null') @@ -98,7 +98,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col stake_amount, amount, amount_requested, open_date, close_date, open_order_id, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag, + max_rate, min_rate, sell_reason, sell_order_status, strategy, enter_tag, timeframe, open_trade_value, close_profit_abs, trading_mode, leverage, isolated_liq, is_short, interest_rate, funding_fees @@ -116,7 +116,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, {sell_order_status} sell_order_status, - {strategy} strategy, {buy_tag} buy_tag, {timeframe} timeframe, + {strategy} strategy, {enter_tag} enter_tag, {timeframe} timeframe, {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs, {trading_mode} trading_mode, {leverage} leverage, {isolated_liq} isolated_liq, {is_short} is_short, {interest_rate} interest_rate, diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f73edbe02..3314f8204 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -264,7 +264,7 @@ class LocalTrade(): sell_reason: str = '' sell_order_status: str = '' strategy: str = '' - buy_tag: Optional[str] = None + enter_tag: Optional[str] = None timeframe: Optional[int] = None trading_mode: TradingMode = TradingMode.SPOT @@ -280,6 +280,14 @@ class LocalTrade(): # Futures properties funding_fees: Optional[float] = None + @property + def buy_tag(self) -> Optional[str]: + """ + Compatibility between buy_tag (old) and enter_tag (new) + Consider buy_tag deprecated + """ + return self.enter_tag + @property def has_no_leverage(self) -> bool: """Returns true if this is a non-leverage, non-short trade""" @@ -389,8 +397,8 @@ class LocalTrade(): 'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None, 'stake_amount': round(self.stake_amount, 8), 'strategy': self.strategy, - 'buy_tag': self.buy_tag, - 'enter_tag': self.buy_tag, + 'buy_tag': self.enter_tag, + 'enter_tag': self.enter_tag, 'timeframe': self.timeframe, 'fee_open': self.fee_open, @@ -929,7 +937,7 @@ class Trade(_DECL_BASE, LocalTrade): sell_reason = Column(String(100), nullable=True) sell_order_status = Column(String(100), nullable=True) strategy = Column(String(100), nullable=True) - buy_tag = Column(String(100), nullable=True) + enter_tag = Column(String(100), nullable=True) timeframe = Column(Integer, nullable=True) trading_mode = Column(Enum(TradingMode), nullable=True) @@ -1100,7 +1108,7 @@ class Trade(_DECL_BASE, LocalTrade): ] @staticmethod - def get_buy_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]: + def get_enter_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, based on buy tag performance Can either be average for all pairs or a specific pair provided @@ -1111,25 +1119,25 @@ class Trade(_DECL_BASE, LocalTrade): if(pair is not None): filters.append(Trade.pair == pair) - buy_tag_perf = Trade.query.with_entities( - Trade.buy_tag, + enter_tag_perf = Trade.query.with_entities( + Trade.enter_tag, func.sum(Trade.close_profit).label('profit_sum'), func.sum(Trade.close_profit_abs).label('profit_sum_abs'), func.count(Trade.pair).label('count') ).filter(*filters)\ - .group_by(Trade.buy_tag) \ + .group_by(Trade.enter_tag) \ .order_by(desc('profit_sum_abs')) \ .all() return [ { - 'buy_tag': buy_tag if buy_tag is not None else "Other", + 'enter_tag': enter_tag if enter_tag is not None else "Other", 'profit_ratio': profit, 'profit_pct': round(profit * 100, 2), 'profit_abs': profit_abs, 'count': count } - for buy_tag, profit, profit_abs, count in buy_tag_perf + for enter_tag, profit, profit_abs, count in enter_tag_perf ] @staticmethod @@ -1179,7 +1187,7 @@ class Trade(_DECL_BASE, LocalTrade): mix_tag_perf = Trade.query.with_entities( Trade.id, - Trade.buy_tag, + Trade.enter_tag, Trade.sell_reason, func.sum(Trade.close_profit).label('profit_sum'), func.sum(Trade.close_profit_abs).label('profit_sum_abs'), @@ -1190,12 +1198,12 @@ class Trade(_DECL_BASE, LocalTrade): .all() return_list: List[Dict] = [] - for id, buy_tag, sell_reason, profit, profit_abs, count in mix_tag_perf: - buy_tag = buy_tag if buy_tag is not None else "Other" + for id, enter_tag, sell_reason, profit, profit_abs, count in mix_tag_perf: + enter_tag = enter_tag if enter_tag is not None else "Other" sell_reason = sell_reason if sell_reason is not None else "Other" - if(sell_reason is not None and buy_tag is not None): - mix_tag = buy_tag + " " + sell_reason + if(sell_reason is not None and enter_tag is not None): + mix_tag = enter_tag + " " + sell_reason i = 0 if not any(item["mix_tag"] == mix_tag for item in return_list): return_list.append({'mix_tag': mix_tag, diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9a47cd112..2a1445e1a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -782,12 +782,12 @@ class RPC: return pair_rates - def _rpc_buy_tag_performance(self, pair: Optional[str]) -> List[Dict[str, Any]]: + def _rpc_enter_tag_performance(self, pair: Optional[str]) -> List[Dict[str, Any]]: """ Handler for buy tag performance. Shows a performance statistic from finished trades """ - buy_tags = Trade.get_buy_tag_performance(pair) + buy_tags = Trade.get_enter_tag_performance(pair) return buy_tags diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 3bcc6adf2..35811e1be 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -154,7 +154,7 @@ class Telegram(RPCHandler): CommandHandler('trades', self._trades), CommandHandler('delete', self._delete_trade), CommandHandler('performance', self._performance), - CommandHandler('buys', self._buy_tag_performance), + CommandHandler(['buys', 'entries'], self._enter_tag_performance), CommandHandler('sells', self._sell_reason_performance), CommandHandler('mix_tags', self._mix_tag_performance), CommandHandler('stats', self._stats), @@ -182,7 +182,8 @@ class Telegram(RPCHandler): CallbackQueryHandler(self._profit, pattern='update_profit'), CallbackQueryHandler(self._balance, pattern='update_balance'), CallbackQueryHandler(self._performance, pattern='update_performance'), - CallbackQueryHandler(self._buy_tag_performance, pattern='update_buy_tag_performance'), + CallbackQueryHandler(self._enter_tag_performance, + pattern='update_enter_tag_performance'), CallbackQueryHandler(self._sell_reason_performance, pattern='update_sell_reason_performance'), CallbackQueryHandler(self._mix_tag_performance, pattern='update_mix_tag_performance'), @@ -972,7 +973,7 @@ class Telegram(RPCHandler): self._send_msg(str(e)) @authorized_only - def _buy_tag_performance(self, update: Update, context: CallbackContext) -> None: + def _enter_tag_performance(self, update: Update, context: CallbackContext) -> None: """ Handler for /buys PAIR . Shows a performance statistic from finished trades @@ -985,7 +986,7 @@ class Telegram(RPCHandler): if context.args and isinstance(context.args[0], str): pair = context.args[0] - trades = self._rpc._rpc_buy_tag_performance(pair) + trades = self._rpc._rpc_enter_tag_performance(pair) output = "Buy Tag Performance:\n" for i, trade in enumerate(trades): stat_line = ( @@ -1001,7 +1002,7 @@ class Telegram(RPCHandler): output += stat_line self._send_msg(output, parse_mode=ParseMode.HTML, - reload_able=True, callback_path="update_buy_tag_performance", + reload_able=True, callback_path="update_enter_tag_performance", query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -1277,7 +1278,8 @@ class Telegram(RPCHandler): " *table :* `will display trades in a table`\n" " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" - "*/buys :* `Shows the buy_tag performance`\n" + # TODO-lev: Update commands and help (?) + "*/buys :* `Shows the enter_tag performance`\n" "*/sells :* `Shows the sell reason performance`\n" "*/mix_tags :* `Shows combined buy tag + sell reason performance`\n" "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" diff --git a/tests/conftest.py b/tests/conftest.py index e184903d1..6a85f5de2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -215,8 +215,6 @@ def patch_get_signal( ) -> None: """ :param mocker: mocker to patch IStrategy class - :param value: which value IStrategy.get_signal() must return - (buy, sell, buy_tag) :return: None """ # returns (Signal-direction, signaname) diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 0ad01e72f..a245033b9 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -102,7 +102,7 @@ def mock_trade_2(fee, is_short: bool): open_order_id=f'dry_run_sell_{direc(is_short)}_12345', strategy='StrategyTestV3', timeframe=5, - buy_tag='TEST1', + enter_tag='TEST1', sell_reason='sell_signal', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), @@ -258,7 +258,7 @@ def mock_trade_5(fee, is_short: bool): open_rate=0.123, exchange='binance', strategy='SampleStrategy', - buy_tag='TEST1', + enter_tag='TEST1', stoploss_order_id=f'prod_stoploss_{direc(is_short)}_3455', timeframe=5, is_short=is_short @@ -314,7 +314,7 @@ def mock_trade_6(fee, is_short: bool): open_rate=0.15, exchange='binance', strategy='SampleStrategy', - buy_tag='TEST2', + enter_tag='TEST2', open_order_id=f"prod_sell_{direc(is_short)}_6", timeframe=5, is_short=is_short diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 599450b57..6db88d123 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -621,6 +621,6 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.sell_reason == trade.sell_reason.value - assert res.buy_tag == trade.enter_tag + assert res.enter_tag == trade.enter_tag assert res.open_date == _get_frame_time_from_offset(trade.open_tick) assert res.close_date == _get_frame_time_from_offset(trade.close_tick) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 19aa56ef4..e50c88b46 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -698,7 +698,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'min_rate': [0.10370188, 0.10300000000000001], 'max_rate': [0.10501, 0.1038888], 'is_open': [False, False], - 'buy_tag': [None, None] + 'enter_tag': [None, None] }) pd.testing.assert_frame_equal(results, expected) data_pair = processed[pair] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 9821c9468..5996fc1f7 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -70,6 +70,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'max_rate': ANY, 'strategy': ANY, 'buy_tag': ANY, + 'enter_tag': ANY, 'timeframe': 5, 'open_order_id': ANY, 'close_date': None, @@ -143,6 +144,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'max_rate': ANY, 'strategy': ANY, 'buy_tag': ANY, + 'enter_tag': ANY, 'timeframe': ANY, 'open_order_id': ANY, 'close_date': None, @@ -842,8 +844,8 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, assert prec_satoshi(res[0]['profit_pct'], 6.2) -def test_buy_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: +def test_enter_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, + limit_sell_order, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -869,23 +871,23 @@ def test_buy_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, trade.close_date = datetime.utcnow() trade.is_open = False - res = rpc._rpc_buy_tag_performance(None) + res = rpc._rpc_enter_tag_performance(None) assert len(res) == 1 - assert res[0]['buy_tag'] == 'Other' + assert res[0]['enter_tag'] == 'Other' assert res[0]['count'] == 1 assert prec_satoshi(res[0]['profit_pct'], 6.2) - trade.buy_tag = "TEST_TAG" - res = rpc._rpc_buy_tag_performance(None) + trade.enter_tag = "TEST_TAG" + res = rpc._rpc_enter_tag_performance(None) assert len(res) == 1 - assert res[0]['buy_tag'] == 'TEST_TAG' + assert res[0]['enter_tag'] == 'TEST_TAG' assert res[0]['count'] == 1 assert prec_satoshi(res[0]['profit_pct'], 6.2) -def test_buy_tag_performance_handle_2(mocker, default_conf, markets, fee): +def test_enter_tag_performance_handle_2(mocker, default_conf, markets, fee): mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -896,21 +898,21 @@ def test_buy_tag_performance_handle_2(mocker, default_conf, markets, fee): create_mock_trades(fee) rpc = RPC(freqtradebot) - res = rpc._rpc_buy_tag_performance(None) + res = rpc._rpc_enter_tag_performance(None) assert len(res) == 2 - assert res[0]['buy_tag'] == 'TEST1' + assert res[0]['enter_tag'] == 'TEST1' assert res[0]['count'] == 1 assert prec_satoshi(res[0]['profit_pct'], 0.5) - assert res[1]['buy_tag'] == 'Other' + assert res[1]['enter_tag'] == 'Other' assert res[1]['count'] == 1 assert prec_satoshi(res[1]['profit_pct'], 1.0) # Test for a specific pair - res = rpc._rpc_buy_tag_performance('ETC/BTC') + res = rpc._rpc_enter_tag_performance('ETC/BTC') assert len(res) == 1 assert res[0]['count'] == 1 - assert res[0]['buy_tag'] == 'TEST1' + assert res[0]['enter_tag'] == 'TEST1' assert prec_satoshi(res[0]['profit_pct'], 0.5) @@ -1020,7 +1022,7 @@ def test_mix_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, assert res[0]['count'] == 1 assert prec_satoshi(res[0]['profit_pct'], 6.2) - trade.buy_tag = "TESTBUY" + trade.enter_tag = "TESTBUY" trade.sell_reason = "TESTSELL" res = rpc._rpc_mix_tag_performance(None) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f2096c0c0..332478c64 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -957,6 +957,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'sell_order_status': None, 'strategy': CURRENT_TEST_STRATEGY, 'buy_tag': None, + 'enter_tag': None, 'timeframe': 5, 'exchange': 'binance', } @@ -1115,6 +1116,7 @@ def test_api_forcebuy(botclient, mocker, fee): 'sell_order_status': None, 'strategy': CURRENT_TEST_STRATEGY, 'buy_tag': None, + 'enter_tag': None, 'timeframe': 5, 'exchange': 'binance', } diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index f64f05ddd..c01820599 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -189,6 +189,7 @@ def test_telegram_status(default_conf, update, mocker) -> None: 'amount': 90.99181074, 'stake_amount': 90.99181074, 'buy_tag': None, + 'enter_tag': None, 'close_profit_ratio': None, 'profit': -0.0059, 'profit_ratio': -0.0059, @@ -954,6 +955,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'buy_tag': ANY, + 'enter_tag': ANY, 'sell_reason': SellType.FORCE_SELL.value, 'open_date': ANY, 'close_date': ANY, @@ -1018,6 +1020,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'buy_tag': ANY, + 'enter_tag': ANY, 'sell_reason': SellType.FORCE_SELL.value, 'open_date': ANY, 'close_date': ANY, @@ -1072,6 +1075,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'buy_tag': ANY, + 'enter_tag': ANY, 'sell_reason': SellType.FORCE_SELL.value, 'open_date': ANY, 'close_date': ANY, @@ -1235,14 +1239,14 @@ def test_buy_tag_performance_handle(default_conf, update, ticker, fee, # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) - trade.buy_tag = "TESTBUY" + trade.enter_tag = "TESTBUY" # Simulate fulfilled LIMIT_SELL order for trade trade.update(limit_sell_order) trade.close_date = datetime.utcnow() trade.is_open = False - telegram._buy_tag_performance(update=update, context=MagicMock()) + telegram._enter_tag_performance(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'Buy Tag Performance' in msg_mock.call_args_list[0][0][0] assert 'TESTBUY\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] @@ -1297,7 +1301,7 @@ def test_mix_tag_performance_handle(default_conf, update, ticker, fee, # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) - trade.buy_tag = "TESTBUY" + trade.enter_tag = "TESTBUY" trade.sell_reason = "TESTSELL" # Simulate fulfilled LIMIT_SELL order for trade @@ -1598,7 +1602,7 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None: msg = { 'type': RPCMessageType.BUY, 'trade_id': 1, - 'buy_tag': 'buy_signal_01', + 'enter_tag': 'buy_signal_01', 'exchange': 'Binance', 'pair': 'ETH/BTC', 'limit': 1.099e-05, @@ -1616,7 +1620,7 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None: telegram.send_msg(msg) assert msg_mock.call_args[0][0] \ == '\N{LARGE BLUE CIRCLE} *Binance:* Buying ETH/BTC (#1)\n' \ - '*Buy Tag:* `buy_signal_01`\n' \ + '*Enter Tag:* `buy_signal_01`\n' \ '*Amount:* `1333.33333333`\n' \ '*Open Rate:* `0.00001099`\n' \ '*Current Rate:* `0.00001099`\n' \ @@ -1691,7 +1695,7 @@ def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: telegram.send_msg({ 'type': RPCMessageType.BUY_FILL, 'trade_id': 1, - 'buy_tag': 'buy_signal_01', + 'enter_tag': 'buy_signal_01', 'exchange': 'Binance', 'pair': 'ETH/BTC', 'stake_amount': 0.001, @@ -1705,7 +1709,7 @@ def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] \ == '\N{CHECK MARK} *Binance:* Bought ETH/BTC (#1)\n' \ - '*Buy Tag:* `buy_signal_01`\n' \ + '*Enter Tag:* `buy_signal_01`\n' \ '*Amount:* `1333.33333333`\n' \ '*Open Rate:* `0.00001099`\n' \ '*Total:* `(0.00100000 BTC, 12.345 USD)`' @@ -1893,7 +1897,7 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: telegram.send_msg({ 'type': RPCMessageType.BUY, - 'buy_tag': 'buy_signal_01', + 'enter_tag': 'buy_signal_01', 'trade_id': 1, 'exchange': 'Binance', 'pair': 'ETH/BTC', @@ -1908,7 +1912,7 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: 'open_date': arrow.utcnow().shift(hours=-1) }) assert msg_mock.call_args[0][0] == ('\N{LARGE BLUE CIRCLE} *Binance:* Buying ETH/BTC (#1)\n' - '*Buy Tag:* `buy_signal_01`\n' + '*Enter Tag:* `buy_signal_01`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00001099`\n' '*Current Rate:* `0.00001099`\n' @@ -1934,14 +1938,14 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', 'fiat_currency': 'USD', - 'buy_tag': 'buy_signal1', + 'enter_tag': 'buy_signal1', 'sell_reason': SellType.STOP_LOSS.value, 'open_date': arrow.utcnow().shift(hours=-2, minutes=-35, seconds=-3), 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' - '*Buy Tag:* `buy_signal1`\n' + '*Enter Tag:* `buy_signal1`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `2:35:03 (155.1 min)`\n' '*Amount:* `1333.33333333`\n' diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 08b2801f7..54d3a95a0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2858,6 +2858,7 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'amount': amt, 'order_type': 'limit', 'buy_tag': None, + 'enter_tag': None, 'open_rate': open_rate, 'current_rate': 2.01 if is_short else 2.3, 'profit_amount': 0.29554455 if is_short else 5.685, @@ -2914,6 +2915,7 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd 'amount': 29.70297029 if is_short else 30.0, 'order_type': 'limit', 'buy_tag': None, + 'enter_tag': None, 'open_rate': 2.02 if is_short else 2.0, 'current_rate': 2.2 if is_short else 2.0, 'profit_amount': -5.65990099 if is_short else -0.00075, @@ -2991,6 +2993,7 @@ def test_execute_trade_exit_custom_exit_price( 'amount': amount, 'order_type': 'limit', 'buy_tag': None, + 'enter_tag': None, 'open_rate': open_rate, 'current_rate': current_rate, 'profit_amount': profit_amount, @@ -3055,6 +3058,7 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( 'amount': 29.70297029 if is_short else 30.0, 'order_type': 'limit', 'buy_tag': None, + 'enter_tag': None, 'open_rate': 2.02 if is_short else 2.0, 'current_rate': 2.2 if is_short else 2.0, 'profit_amount': -0.3 if is_short else -0.8985, @@ -3308,6 +3312,7 @@ def test_execute_trade_exit_market_order( 'amount': round(amount, 9), 'order_type': 'market', 'buy_tag': None, + 'enter_tag': None, 'open_rate': open_rate, 'current_rate': current_rate, 'profit_amount': profit_amount, diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 9df4f511a..f1401eef1 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1551,7 +1551,7 @@ def test_to_json(default_conf, fee): open_date=arrow.utcnow().shift(hours=-2).datetime, open_rate=0.123, exchange='binance', - buy_tag=None, + enter_tag=None, open_order_id='dry_run_buy_12345' ) result = trade.to_json() @@ -1625,7 +1625,7 @@ def test_to_json(default_conf, fee): close_date=arrow.utcnow().shift(hours=-1).datetime, open_rate=0.123, close_rate=0.125, - buy_tag='buys_signal_001', + enter_tag='buys_signal_001', exchange='binance', ) result = trade.to_json() @@ -2118,7 +2118,7 @@ def test_Trade_object_idem(): 'get_open_order_trades', 'get_trades', 'get_sell_reason_performance', - 'get_buy_tag_performance', + 'get_enter_tag_performance', 'get_mix_tag_performance', ) From 2a84526f04718c84872ce46007d310efb45e0272 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Nov 2021 10:05:56 +0100 Subject: [PATCH 03/55] Remove more buy_tag references --- freqtrade/rpc/api_server/api_schemas.py | 2 +- freqtrade/rpc/rpc.py | 10 +++------- freqtrade/rpc/telegram.py | 2 +- tests/rpc/test_rpc_telegram.py | 16 ++++++++-------- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index db4378a51..cc92cb81e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -184,7 +184,7 @@ class TradeSchema(BaseModel): amount_requested: float stake_amount: float strategy: str - buy_tag: Optional[str] + buy_tag: Optional[str] # Deprecated enter_tag: Optional[str] timeframe: int fee_open: Optional[float] diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2a1445e1a..7242bec2a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -787,22 +787,18 @@ class RPC: Handler for buy tag performance. Shows a performance statistic from finished trades """ - buy_tags = Trade.get_enter_tag_performance(pair) - - return buy_tags + return Trade.get_enter_tag_performance(pair) def _rpc_sell_reason_performance(self, pair: Optional[str]) -> List[Dict[str, Any]]: """ Handler for sell reason performance. Shows a performance statistic from finished trades """ - sell_reasons = Trade.get_sell_reason_performance(pair) - - return sell_reasons + return Trade.get_sell_reason_performance(pair) def _rpc_mix_tag_performance(self, pair: Optional[str]) -> List[Dict[str, Any]]: """ - Handler for mix tag (buy_tag + sell_reason) performance. + Handler for mix tag (enter_tag + sell_reason) performance. Shows a performance statistic from finished trades """ mix_tags = Trade.get_mix_tag_performance(pair) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 35811e1be..3e7332db7 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -272,7 +272,7 @@ class Telegram(RPCHandler): f"{'Sold' if is_fill else 'Selling'} {msg['pair']} (#{msg['trade_id']})\n" f"*{'Profit' if is_fill else 'Unrealized Profit'}:* " f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n" - f"*Buy Tag:* `{msg['buy_tag']}`\n" + f"*Enter Tag:* `{msg['enter_tag']}`\n" f"*Sell Reason:* `{msg['sell_reason']}`\n" f"*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`\n" f"*Amount:* `{msg['amount']:.8f}`\n") diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index c01820599..35d7b365f 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -93,7 +93,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: message_str = ("rpc.telegram is listening for following commands: [['status'], ['profit'], " "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], ['trades'], " - "['delete'], ['performance'], ['buys'], ['sells'], ['mix_tags'], " + "['delete'], ['performance'], ['buys', 'entries'], ['sells'], ['mix_tags'], " "['stats'], ['daily'], ['weekly'], ['monthly'], " "['count'], ['locks'], ['unlock', 'delete_locks'], " "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], " @@ -1648,7 +1648,7 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: telegram.send_msg({ 'type': RPCMessageType.BUY_CANCEL, - 'buy_tag': 'buy_signal_01', + 'enter_tag': 'buy_signal_01', 'trade_id': 1, 'exchange': 'Binance', 'pair': 'ETH/BTC', @@ -1736,7 +1736,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', 'fiat_currency': 'USD', - 'buy_tag': 'buy_signal1', + 'enter_tag': 'buy_signal1', 'sell_reason': SellType.STOP_LOSS.value, 'open_date': arrow.utcnow().shift(hours=-1), 'close_date': arrow.utcnow(), @@ -1744,7 +1744,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] \ == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' - '*Buy Tag:* `buy_signal1`\n' + '*Enter Tag:* `buy_signal1`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `1:00:00 (60.0 min)`\n' '*Amount:* `1333.33333333`\n' @@ -1768,7 +1768,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'profit_amount': -0.05746268, 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', - 'buy_tag': 'buy_signal1', + 'enter_tag': 'buy_signal1', 'sell_reason': SellType.STOP_LOSS.value, 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), 'close_date': arrow.utcnow(), @@ -1776,7 +1776,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] \ == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' - '*Buy Tag:* `buy_signal1`\n' + '*Enter Tag:* `buy_signal1`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' '*Amount:* `1333.33333333`\n' @@ -1839,7 +1839,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker) -> None: 'profit_amount': -0.05746268, 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', - 'buy_tag': 'buy_signal1', + 'enter_tag': 'buy_signal1', 'sell_reason': SellType.STOP_LOSS.value, 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), 'close_date': arrow.utcnow(), @@ -1847,7 +1847,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] \ == ('\N{WARNING SIGN} *Binance:* Sold KEY/ETH (#1)\n' '*Profit:* `-57.41%`\n' - '*Buy Tag:* `buy_signal1`\n' + '*Enter Tag:* `buy_signal1`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' '*Amount:* `1333.33333333`\n' From 192ac88314f6b3d7f14862b7da82b65b1b761f68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Nov 2021 10:16:18 +0100 Subject: [PATCH 04/55] Update optimize-reports to enter_tag wording --- freqtrade/optimize/optimize_reports.py | 16 +++++++++------- freqtrade/persistence/migrations.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index c4002fcbe..2a1309548 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -422,8 +422,8 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], starting_balance=starting_balance, results=results, skip_nan=False) - buy_tag_results = generate_tag_metrics("buy_tag", starting_balance=starting_balance, - results=results, skip_nan=False) + enter_tag_results = generate_tag_metrics("enter_tag", starting_balance=starting_balance, + results=results, skip_nan=False) sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades, results=results) @@ -448,7 +448,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], 'best_pair': best_pair, 'worst_pair': worst_pair, 'results_per_pair': pair_results, - 'results_per_buy_tag': buy_tag_results, + 'results_per_enter_tag': enter_tag_results, 'sell_reason_summary': sell_reason_stats, 'left_open_trades': left_open_results, # 'days_breakdown_stats': days_breakdown_stats, @@ -628,7 +628,7 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr :param stake_currency: stake-currency - used to correctly name headers :return: pretty printed table with tabulate as string """ - if(tag_type == "buy_tag"): + if(tag_type == "enter_tag"): headers = _get_line_header("TAG", stake_currency) else: headers = _get_line_header_sell("TAG", stake_currency) @@ -797,10 +797,12 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) print(table) - if results.get('results_per_buy_tag') is not None: + if (results.get('results_per_enter_tag') is not None + or results.get('results_per_buy_tag') is not None): + # results_per_buy_tag is deprecated and should be removed 2 versions after short golive. table = text_table_tags( - "buy_tag", - results['results_per_buy_tag'], + "enter_tag", + results.get('results_per_enter_tag', results.get('results_per_buy_tag')), stake_currency=stake_currency) if isinstance(table, str) and len(table) > 0: diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 212499df3..99b8f0925 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -180,7 +180,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: table_back_name = get_backup_name(tabs, 'trades_bak') # Check for latest column - if not has_column(cols, 'funding_fees'): + if not has_column(cols, 'enter_tag'): logger.info(f'Running database migration for trades - backup: {table_back_name}') migrate_trades_table(decl_base, inspector, engine, table_back_name, cols) # Reread columns - the above recreated the table! From c23ca35d23721eb165e485fe17c841560573e03e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 Nov 2021 07:13:39 +0100 Subject: [PATCH 05/55] Update ARMHF image to 3.9 --- Dockerfile | 2 +- docker/Dockerfile.armhf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f7e26efe3..8f5b85698 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.7-slim-buster as base +FROM python:3.9.9-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index f9827774e..16f2aebcd 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.7.10-slim-buster as base +FROM python:3.9.9-slim-bullseye as base # Setup env ENV LANG C.UTF-8 From e8feac367401952455433ee351066cdecef689f0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 Nov 2021 20:02:07 +0100 Subject: [PATCH 06/55] Improve tests for pair_to_filename --- tests/test_misc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_misc.py b/tests/test_misc.py index 221c7b712..75f4c8622 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -67,6 +67,9 @@ def test_file_load_json(mocker, testdatadir) -> None: @pytest.mark.parametrize("pair,expected_result", [ ("ETH/BTC", 'ETH_BTC'), + ("ETH/USDT", 'ETH_USDT'), + ("ETH/USDT:USDT", 'ETH_USDT_USDT'), # swap with USDT as settlement currency + ("ETH/USDT:USDT-210625", 'ETH_USDT_USDT-210625'), # expiring futures ("Fabric Token/ETH", 'Fabric_Token_ETH'), ("ETHH20", 'ETHH20'), (".XBTBON2H", '_XBTBON2H'), From 65906d330f3c6291a22c63d9592f2e11b42aeee4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 Nov 2021 20:07:54 +0100 Subject: [PATCH 07/55] Improve tests for pair_to_filename --- tests/test_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index 75f4c8622..de3f368e9 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -69,7 +69,7 @@ def test_file_load_json(mocker, testdatadir) -> None: ("ETH/BTC", 'ETH_BTC'), ("ETH/USDT", 'ETH_USDT'), ("ETH/USDT:USDT", 'ETH_USDT_USDT'), # swap with USDT as settlement currency - ("ETH/USDT:USDT-210625", 'ETH_USDT_USDT-210625'), # expiring futures + ("ETH/USDT:USDT-210625", 'ETH_USDT_USDT_210625'), # expiring futures ("Fabric Token/ETH", 'Fabric_Token_ETH'), ("ETHH20", 'ETHH20'), (".XBTBON2H", '_XBTBON2H'), From 338fe333a9bac02cc4ffc80eb8e0505ae7da94de Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 Nov 2021 20:11:04 +0100 Subject: [PATCH 08/55] Allow forcebuy to specify order_type --- freqtrade/freqtradebot.py | 10 ++++------ freqtrade/rpc/api_server/api_schemas.py | 22 ++++++++++++++++------ freqtrade/rpc/api_server/api_v1.py | 5 +++-- freqtrade/rpc/rpc.py | 8 ++++++-- tests/rpc/test_rpc.py | 2 +- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index db0453cd7..57d5e0528 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -466,8 +466,8 @@ class FreqtradeBot(LoggingMixin): logger.info(f"Bids to asks delta for {pair} does not satisfy condition.") return False - def execute_entry(self, pair: str, stake_amount: float, price: Optional[float] = None, - forcebuy: bool = False, buy_tag: Optional[str] = None) -> bool: + def execute_entry(self, pair: str, stake_amount: float, price: Optional[float] = None, *, + order_type: Optional[str] = None, buy_tag: Optional[str] = None) -> bool: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY @@ -510,10 +510,8 @@ class FreqtradeBot(LoggingMixin): f"{stake_amount} ...") amount = stake_amount / enter_limit_requested - order_type = self.strategy.order_types['buy'] - if forcebuy: - # Forcebuy can define a different ordertype - order_type = self.strategy.order_types.get('forcebuy', order_type) + if not order_type: + order_type = self.strategy.order_types['buy'] if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 268d50fdb..ed483b18d 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -1,4 +1,5 @@ from datetime import date, datetime +from enum import Enum from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel @@ -131,13 +132,21 @@ class UnfilledTimeout(BaseModel): exit_timeout_count: Optional[int] +class OrderTypeValues(Enum): + limit = 'limit' + market = 'market' + + class Config: + use_enum_values = True + + class OrderTypes(BaseModel): - buy: str - sell: str - emergencysell: Optional[str] - forcesell: Optional[str] - forcebuy: Optional[str] - stoploss: str + buy: OrderTypeValues + sell: OrderTypeValues + emergencysell: Optional[OrderTypeValues] + forcesell: Optional[OrderTypeValues] + forcebuy: Optional[OrderTypeValues] + stoploss: OrderTypeValues stoploss_on_exchange: bool stoploss_on_exchange_interval: Optional[int] @@ -274,6 +283,7 @@ class Logs(BaseModel): class ForceBuyPayload(BaseModel): pair: str price: Optional[float] + ordertype: Optional[OrderTypeValues] class ForceSellPayload(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 0467e4705..6fc135820 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -29,7 +29,8 @@ logger = logging.getLogger(__name__) # API version # Pre-1.1, no version was provided # Version increments should happen in "small" steps (1.1, 1.12, ...) unless big changes happen. -API_VERSION = 1.1 +# 1.11: forcebuy accepts new option with ordertype +API_VERSION = 1.11 # Public API, requires no auth. router_public = APIRouter() @@ -129,7 +130,7 @@ def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(g @router.post('/forcebuy', response_model=ForceBuyResponse, tags=['trading']) def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): - trade = rpc._rpc_forcebuy(payload.pair, payload.price) + trade = rpc._rpc_forcebuy(payload.pair, payload.price, payload.ordertype) if trade: return ForceBuyResponse.parse_obj(trade.to_json()) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 28585e4e8..fc1c0c777 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -692,7 +692,8 @@ class RPC: self._freqtrade.wallets.update() return {'result': f'Created sell order for trade {trade_id}.'} - def _rpc_forcebuy(self, pair: str, price: Optional[float]) -> Optional[Trade]: + def _rpc_forcebuy(self, pair: str, price: Optional[float], + order_type: Optional[str] = None) -> Optional[Trade]: """ Handler for forcebuy Buys a pair trade at the given or current price @@ -720,7 +721,10 @@ class RPC: stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair) # execute buy - if self._freqtrade.execute_entry(pair, stakeamount, price, forcebuy=True): + if not order_type: + order_type = self._freqtrade.strategy.order_types.get( + 'forcebuy', self._freqtrade.strategy.order_types['buy']) + if self._freqtrade.execute_entry(pair, stakeamount, price, order_type=order_type): Trade.commit() trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() return trade diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 2852ada81..b6fe1c691 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1093,7 +1093,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order_open) -> with pytest.raises(RPCException, match=r'position for ETH/BTC already open - id: 1'): rpc._rpc_forcebuy(pair, 0.0001) pair = 'XRP/BTC' - trade = rpc._rpc_forcebuy(pair, 0.0001) + trade = rpc._rpc_forcebuy(pair, 0.0001, order_type='limit') assert isinstance(trade, Trade) assert trade.pair == pair assert trade.open_rate == 0.0001 From 0d1e84cf553822001dfb126a9cb1af0289f30f44 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 25 Nov 2021 16:00:10 +0900 Subject: [PATCH 09/55] Add more words Because apparently, we get at least 1 question about this everyday in Discord --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 49957c2bb..15613cb06 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,7 +89,8 @@ class Backtesting: self.init_backtest_detail() self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: - raise OperationalException("VolumePairList not allowed for backtesting.") + raise OperationalException("VolumePairList not allowed for backtesting. " + "Please use StaticPairlist.") if 'PerformanceFilter' in self.pairlists.name_list: raise OperationalException("PerformanceFilter not allowed for backtesting.") From 0c629fc951f484019c4d563e8792f3c314088755 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 25 Nov 2021 16:03:29 +0900 Subject: [PATCH 10/55] Update test_backtesting.py --- tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index ab7aa74a1..fb9282c2d 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -438,7 +438,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> Backtesting(default_conf) default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] - with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'): + with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): Backtesting(default_conf) default_conf.update({ @@ -470,7 +470,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['timerange'] = '20180101-20180102' default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] - with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'): + with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): Backtesting(default_conf) default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] From c23d90e2b84434a796ed9822ed00630df268c348 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 25 Nov 2021 16:56:56 +0900 Subject: [PATCH 11/55] Update test_backtesting.py --- tests/optimize/test_backtesting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index fb9282c2d..548d13b31 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -438,7 +438,8 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> Backtesting(default_conf) default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] - with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): + with pytest.raises(OperationalException, + match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): Backtesting(default_conf) default_conf.update({ @@ -470,7 +471,8 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['timerange'] = '20180101-20180102' default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] - with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): + with pytest.raises(OperationalException, + match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): Backtesting(default_conf) default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] From 5307d2bf3bfe3c8ed37143b9a9eda1cdd90183b3 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 25 Nov 2021 17:04:04 +0900 Subject: [PATCH 12/55] Trimming the sentence --- freqtrade/optimize/backtesting.py | 2 +- tests/optimize/test_backtesting.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 15613cb06..b16aced28 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -90,7 +90,7 @@ class Backtesting: self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting. " - "Please use StaticPairlist.") + "Use StaticPairlist.") if 'PerformanceFilter' in self.pairlists.name_list: raise OperationalException("PerformanceFilter not allowed for backtesting.") diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 548d13b31..2a017ed7d 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -439,7 +439,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] with pytest.raises(OperationalException, - match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): + match='VolumePairList not allowed for backtesting. Use StaticPairlist.'): Backtesting(default_conf) default_conf.update({ @@ -472,7 +472,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] with pytest.raises(OperationalException, - match='VolumePairList not allowed for backtesting. Please use StaticPairlist.'): + match='VolumePairList not allowed for backtesting. Use StaticPairlist.'): Backtesting(default_conf) default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] From f4bc30c927ea544560c801dec398b427eef9f3f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 26 Nov 2021 06:23:29 +0100 Subject: [PATCH 13/55] Update docs to include "vpn/ssh" section --- docs/rest-api.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/rest-api.md b/docs/rest-api.md index 7299e0282..8c2599cbc 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -38,6 +38,11 @@ Sample configuration: !!! Danger "Security warning" By default, the configuration listens on localhost only (so it's not reachable from other systems). We strongly recommend to not expose this API to the internet and choose a strong, unique password, since others will potentially be able to control your bot. +??? Note "API/UI Access on a remote servers" + If you're running on a VPS, you should consider using either a ssh tunnel, or setup a VPN (openVPN, wireguard) to connect to your bot. + This will ensure that freqUI is not directly exposed to the internet, which is not recommended for security reasons (freqUI does not support https out of the box). + Setup of these tools is not part of this tutorial, however many good tutorials can be found on the internet. + You can then access the API by going to `http://127.0.0.1:8080/api/v1/ping` in a browser to check if the API is running correctly. This should return the response: From 897788de17dc70c78d44fba6dfeb1366ab167f11 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 26 Nov 2021 06:27:06 +0100 Subject: [PATCH 14/55] Reformulate exception to be "nicer" --- freqtrade/optimize/backtesting.py | 2 +- tests/optimize/test_backtesting.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b16aced28..219a4f069 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -90,7 +90,7 @@ class Backtesting: self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting. " - "Use StaticPairlist.") + "Please use StaticPairlist instead.") if 'PerformanceFilter' in self.pairlists.name_list: raise OperationalException("PerformanceFilter not allowed for backtesting.") diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 2a017ed7d..f5e182c1d 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -439,7 +439,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] with pytest.raises(OperationalException, - match='VolumePairList not allowed for backtesting. Use StaticPairlist.'): + match=r'VolumePairList not allowed for backtesting\..*StaticPairlist.*'): Backtesting(default_conf) default_conf.update({ @@ -472,7 +472,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] with pytest.raises(OperationalException, - match='VolumePairList not allowed for backtesting. Use StaticPairlist.'): + match=r'VolumePairList not allowed for backtesting\..*StaticPairlist.*'): Backtesting(default_conf) default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] From 80ed5283b24096168a441f0890fabb2075c5d929 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 09:10:18 +0100 Subject: [PATCH 15/55] Add forcesell market/limit distinction --- freqtrade/freqtradebot.py | 18 ++++++++---------- freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/api_server/api_v1.py | 6 +++--- freqtrade/rpc/rpc.py | 10 +++++++--- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 57d5e0528..a6d1b36b9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -467,7 +467,7 @@ class FreqtradeBot(LoggingMixin): return False def execute_entry(self, pair: str, stake_amount: float, price: Optional[float] = None, *, - order_type: Optional[str] = None, buy_tag: Optional[str] = None) -> bool: + ordertype: Optional[str] = None, buy_tag: Optional[str] = None) -> bool: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY @@ -510,8 +510,7 @@ class FreqtradeBot(LoggingMixin): f"{stake_amount} ...") amount = stake_amount / enter_limit_requested - if not order_type: - order_type = self.strategy.order_types['buy'] + order_type = ordertype or self.strategy.order_types['buy'] if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, @@ -866,7 +865,7 @@ class FreqtradeBot(LoggingMixin): logger.info( f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}. ' f'Tag: {exit_tag if exit_tag is not None else "None"}') - self.execute_trade_exit(trade, exit_rate, should_sell, exit_tag) + self.execute_trade_exit(trade, exit_rate, should_sell, exit_tag=exit_tag) return True return False @@ -1079,7 +1078,10 @@ class FreqtradeBot(LoggingMixin): trade: Trade, limit: float, sell_reason: SellCheckTuple, - exit_tag: Optional[str] = None) -> bool: + *, + exit_tag: Optional[str] = None, + ordertype: Optional[str] = None, + ) -> bool: """ Executes a trade exit for the given trade and limit :param trade: Trade instance @@ -1117,14 +1119,10 @@ class FreqtradeBot(LoggingMixin): except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}") - order_type = self.strategy.order_types[sell_type] + order_type = ordertype or self.strategy.order_types[sell_type] if sell_reason.sell_type == SellType.EMERGENCY_SELL: # Emergency sells (default to market!) order_type = self.strategy.order_types.get("emergencysell", "market") - if sell_reason.sell_type == SellType.FORCE_SELL: - # Force sells (default to the sell_type defined in the strategy, - # but we allow this value to be changed) - order_type = self.strategy.order_types.get("forcesell", order_type) amount = self._safe_exit_amount(trade.pair, trade.amount) time_in_force = self.strategy.order_time_in_force['sell'] diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index ed483b18d..d0e772848 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -288,6 +288,7 @@ class ForceBuyPayload(BaseModel): class ForceSellPayload(BaseModel): tradeid: str + ordertype: Optional[OrderTypeValues] class BlacklistPayload(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 6fc135820..1fd4ca74b 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) # API version # Pre-1.1, no version was provided # Version increments should happen in "small" steps (1.1, 1.12, ...) unless big changes happen. -# 1.11: forcebuy accepts new option with ordertype +# 1.11: forcebuy and forcesell accept ordertype API_VERSION = 1.11 # Public API, requires no auth. @@ -130,7 +130,7 @@ def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(g @router.post('/forcebuy', response_model=ForceBuyResponse, tags=['trading']) def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): - trade = rpc._rpc_forcebuy(payload.pair, payload.price, payload.ordertype) + trade = rpc._rpc_forcebuy(payload.pair, payload.price, payload.ordertype.value) if trade: return ForceBuyResponse.parse_obj(trade.to_json()) @@ -140,7 +140,7 @@ def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): @router.post('/forcesell', response_model=ResultMsg, tags=['trading']) def forcesell(payload: ForceSellPayload, rpc: RPC = Depends(get_rpc)): - return rpc._rpc_forcesell(payload.tradeid) + return rpc._rpc_forcesell(payload.tradeid, payload.ordertype.value) @router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist']) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index fc1c0c777..c21890b7d 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -640,7 +640,7 @@ class RPC: return {'status': 'No more buy will occur from now. Run /reload_config to reset.'} - def _rpc_forcesell(self, trade_id: str) -> Dict[str, str]: + def _rpc_forcesell(self, trade_id: str, ordertype: Optional[str] = None) -> Dict[str, str]: """ Handler for forcesell . Sells the given trade at current price @@ -664,7 +664,11 @@ class RPC: current_rate = self._freqtrade.exchange.get_rate( trade.pair, refresh=False, side="sell") sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL) - self._freqtrade.execute_trade_exit(trade, current_rate, sell_reason) + order_type = ordertype or self._freqtrade.strategy.order_types.get( + "forcesell", self._freqtrade.strategy.order_types["sell"]) + + self._freqtrade.execute_trade_exit( + trade, current_rate, sell_reason, ordertype=order_type) # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: @@ -724,7 +728,7 @@ class RPC: if not order_type: order_type = self._freqtrade.strategy.order_types.get( 'forcebuy', self._freqtrade.strategy.order_types['buy']) - if self._freqtrade.execute_entry(pair, stakeamount, price, order_type=order_type): + if self._freqtrade.execute_entry(pair, stakeamount, price, ordertype=order_type): Trade.commit() trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() return trade From bc52b3db56b02f448ccef3d6a8220b01849fc9ed Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 09:26:14 +0100 Subject: [PATCH 16/55] Properly handle None values via API --- freqtrade/rpc/api_server/api_v1.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 1fd4ca74b..65b6941e2 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -130,7 +130,8 @@ def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(g @router.post('/forcebuy', response_model=ForceBuyResponse, tags=['trading']) def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): - trade = rpc._rpc_forcebuy(payload.pair, payload.price, payload.ordertype.value) + ordertype = payload.ordertype.value if payload.ordertype else None + trade = rpc._rpc_forcebuy(payload.pair, payload.price, ordertype) if trade: return ForceBuyResponse.parse_obj(trade.to_json()) @@ -140,7 +141,8 @@ def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): @router.post('/forcesell', response_model=ResultMsg, tags=['trading']) def forcesell(payload: ForceSellPayload, rpc: RPC = Depends(get_rpc)): - return rpc._rpc_forcesell(payload.tradeid, payload.ordertype.value) + ordertype = payload.ordertype.value if payload.ordertype else None + return rpc._rpc_forcesell(payload.tradeid, ordertype) @router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist']) From 6ca6f62509122d06fcbf4a9d435a8dc96a27bbad Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 09:39:10 +0100 Subject: [PATCH 17/55] Remove duplicate code in optimize_reports --- freqtrade/optimize/optimize_reports.py | 50 +++----------------------- 1 file changed, 4 insertions(+), 46 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index c4002fcbe..dcd6b4e1f 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -46,20 +46,11 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]: '.2f', 'd', 's', 's'] -def _get_line_header(first_column: str, stake_currency: str) -> List[str]: +def _get_line_header(first_column: str, stake_currency: str, direction: str = 'Buys') -> List[str]: """ Generate header lines (goes in line with _generate_result_line()) """ - return [first_column, 'Buys', 'Avg Profit %', 'Cum Profit %', - f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', - 'Win Draw Loss Win%'] - - -def _get_line_header_sell(first_column: str, stake_currency: str) -> List[str]: - """ - Generate header lines (goes in line with _generate_result_line()) - """ - return [first_column, 'Sells', 'Avg Profit %', 'Cum Profit %', + return [first_column, direction, 'Avg Profit %', 'Cum Profit %', f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', 'Win Draw Loss Win%'] @@ -156,7 +147,7 @@ def generate_tag_metrics(tag_type: str, if skip_nan and result['profit_abs'].isnull().all(): continue - tabular_data.append(_generate_tag_result_line(result, starting_balance, tag)) + tabular_data.append(_generate_result_line(result, starting_balance, tag)) # Sort by total profit %: tabular_data = sorted(tabular_data, key=lambda k: k['profit_total_abs'], reverse=True) @@ -168,39 +159,6 @@ def generate_tag_metrics(tag_type: str, return [] -def _generate_tag_result_line(result: DataFrame, starting_balance: int, first_column: str) -> Dict: - """ - Generate one result dict, with "first_column" as key. - """ - profit_sum = result['profit_ratio'].sum() - # (end-capital - starting capital) / starting capital - profit_total = result['profit_abs'].sum() / starting_balance - - return { - 'key': first_column, - 'trades': len(result), - 'profit_mean': result['profit_ratio'].mean() if len(result) > 0 else 0.0, - 'profit_mean_pct': result['profit_ratio'].mean() * 100.0 if len(result) > 0 else 0.0, - 'profit_sum': profit_sum, - 'profit_sum_pct': round(profit_sum * 100.0, 2), - 'profit_total_abs': result['profit_abs'].sum(), - 'profit_total': profit_total, - 'profit_total_pct': round(profit_total * 100.0, 2), - 'duration_avg': str(timedelta( - minutes=round(result['trade_duration'].mean())) - ) if not result.empty else '0:00', - # 'duration_max': str(timedelta( - # minutes=round(result['trade_duration'].max())) - # ) if not result.empty else '0:00', - # 'duration_min': str(timedelta( - # minutes=round(result['trade_duration'].min())) - # ) if not result.empty else '0:00', - 'wins': len(result[result['profit_abs'] > 0]), - 'draws': len(result[result['profit_abs'] == 0]), - 'losses': len(result[result['profit_abs'] < 0]), - } - - def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List[Dict]: """ Generate small table outlining Backtest results @@ -631,7 +589,7 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr if(tag_type == "buy_tag"): headers = _get_line_header("TAG", stake_currency) else: - headers = _get_line_header_sell("TAG", stake_currency) + headers = _get_line_header("TAG", stake_currency, 'Sells') floatfmt = _get_line_floatfmt(stake_currency) output = [ [ From a629777890a9fd2d10cf39eff32ad999964a42ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 09:53:05 +0100 Subject: [PATCH 18/55] Improve test coverage in telegram module --- tests/rpc/test_rpc_telegram.py | 64 +++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index ce3b044be..6c32e59fc 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -24,6 +24,7 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC +from freqtrade.rpc.rpc import RPCException from freqtrade.rpc.telegram import Telegram, authorized_only from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, log_has_re, patch_exchange, patch_get_signal, patch_whitelist) @@ -1186,8 +1187,8 @@ def test_forcebuy_no_pair(default_conf, update, mocker) -> None: assert fbuy_mock.call_count == 1 -def test_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: +def test_telegram_performance_handle(default_conf, update, ticker, fee, + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1216,8 +1217,8 @@ def test_performance_handle(default_conf, update, ticker, fee, assert 'ETH/BTC\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] -def test_buy_tag_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: +def test_telegram_buy_tag_performance_handle(default_conf, update, ticker, fee, + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -1240,15 +1241,27 @@ def test_buy_tag_performance_handle(default_conf, update, ticker, fee, trade.close_date = datetime.utcnow() trade.is_open = False - - telegram._buy_tag_performance(update=update, context=MagicMock()) + context = MagicMock() + telegram._buy_tag_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Buy Tag Performance' in msg_mock.call_args_list[0][0][0] assert 'TESTBUY\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] + context.args = [trade.pair] + telegram._buy_tag_performance(update=update, context=context) + assert msg_mock.call_count == 2 -def test_sell_reason_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: + msg_mock.reset_mock() + mocker.patch('freqtrade.rpc.rpc.RPC._rpc_buy_tag_performance', + side_effect=RPCException('Error')) + telegram._buy_tag_performance(update=update, context=MagicMock()) + + assert msg_mock.call_count == 1 + assert "Error" in msg_mock.call_args_list[0][0][0] + + +def test_telegram_sell_reason_performance_handle(default_conf, update, ticker, fee, + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -1271,15 +1284,27 @@ def test_sell_reason_performance_handle(default_conf, update, ticker, fee, trade.close_date = datetime.utcnow() trade.is_open = False - - telegram._sell_reason_performance(update=update, context=MagicMock()) + context = MagicMock() + telegram._sell_reason_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Sell Reason Performance' in msg_mock.call_args_list[0][0][0] assert 'TESTSELL\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] + context.args = [trade.pair] + + telegram._sell_reason_performance(update=update, context=context) + assert msg_mock.call_count == 2 + + msg_mock.reset_mock() + mocker.patch('freqtrade.rpc.rpc.RPC._rpc_sell_reason_performance', + side_effect=RPCException('Error')) + telegram._sell_reason_performance(update=update, context=MagicMock()) + + assert msg_mock.call_count == 1 + assert "Error" in msg_mock.call_args_list[0][0][0] -def test_mix_tag_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: +def test_telegram_mix_tag_performance_handle(default_conf, update, ticker, fee, + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -1305,12 +1330,25 @@ def test_mix_tag_performance_handle(default_conf, update, ticker, fee, trade.close_date = datetime.utcnow() trade.is_open = False - telegram._mix_tag_performance(update=update, context=MagicMock()) + context = MagicMock() + telegram._mix_tag_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Mix Tag Performance' in msg_mock.call_args_list[0][0][0] assert ('TESTBUY TESTSELL\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0]) + context.args = [trade.pair] + telegram._mix_tag_performance(update=update, context=context) + assert msg_mock.call_count == 2 + + msg_mock.reset_mock() + mocker.patch('freqtrade.rpc.rpc.RPC._rpc_mix_tag_performance', + side_effect=RPCException('Error')) + telegram._mix_tag_performance(update=update, context=MagicMock()) + + assert msg_mock.call_count == 1 + assert "Error" in msg_mock.call_args_list[0][0][0] + def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( From 8c52ba3360667a7eb9611f0ddb83709d48bc14ef Mon Sep 17 00:00:00 2001 From: incrementby1 <91958753+incrementby1@users.noreply.github.com> Date: Sat, 27 Nov 2021 16:21:23 +0100 Subject: [PATCH 19/55] ShuffleFilterDetectLiveMode # Apply seed in backtesting mode to get comparable results, # but not in live modes to get a non-repeating order of pairs during live modes. --- freqtrade/plugins/pairlist/ShuffleFilter.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 4d3dd29e3..834fc9a43 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -18,7 +18,15 @@ class ShuffleFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - self._seed = pairlistconfig.get('seed') + # Apply seed in backtesting mode to get comparable results, + # but not in live modes to get a non-repeating order of pairs during live modes. + if config['runmode'].value in ('live', 'dry_run'): + self._seed = None + logger.info("live mode detected, not applying seed.") + else: + self._seed = pairlistconfig.get('seed') + logger.info("Backtesting mode detected, applying seed value: " + str(self._seed)) + self._random = random.Random(self._seed) @property From 2f0f576fce8af2d41e02136b759382d5746dff78 Mon Sep 17 00:00:00 2001 From: incrementby1 <91958753+incrementby1@users.noreply.github.com> Date: Sat, 27 Nov 2021 16:28:41 +0100 Subject: [PATCH 20/55] Update pairlists.md ShuffleFilter will automatically detect runmodes and apply the `seed` only for backtesting modes - if ad `seed` value is set. --- docs/includes/pairlists.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index bbfe74510..c6b320e62 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -254,10 +254,10 @@ Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - #### ShuffleFilter -Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. +Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. !!! Tip - You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. + You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. ShuffleFilter will automatically detect runmodes and apply the `seed` only for backtesting modes - if a `seed` value is set. #### SpreadFilter From bf8f1045caeafb969b8d6483bb0730c49ee58e89 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 16:46:17 +0100 Subject: [PATCH 21/55] Map binanceusdm to ft binance class --- freqtrade/exchange/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index a4c827e07..fc21c0f02 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -23,6 +23,7 @@ BAD_EXCHANGES = { MAP_EXCHANGE_CHILDCLASS = { 'binanceus': 'binance', 'binanceje': 'binance', + 'binanceusdm': 'binance', } From b90303c9a3d2bdbbb46f4557a6b68bdee463bcd4 Mon Sep 17 00:00:00 2001 From: incrementby1 <91958753+incrementby1@users.noreply.github.com> Date: Sat, 27 Nov 2021 18:26:30 +0100 Subject: [PATCH 22/55] Update ShuffleFilter.py random.Random() is deprecated since 3.9 --- freqtrade/plugins/pairlist/ShuffleFilter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 834fc9a43..3cefee236 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -27,7 +27,8 @@ class ShuffleFilter(IPairList): self._seed = pairlistconfig.get('seed') logger.info("Backtesting mode detected, applying seed value: " + str(self._seed)) - self._random = random.Random(self._seed) + # deprecated since 3.9 + #self._random = random.Random(self._seed) @property def needstickers(self) -> bool: @@ -54,6 +55,7 @@ class ShuffleFilter(IPairList): :return: new whitelist """ # Shuffle is done inplace - self._random.shuffle(pairlist) + random.seed(self._seed) + random.shuffle(pairlist) return pairlist From 409a80176320a7d934eb5159faf35cb2a1ae9989 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 19:30:49 +0100 Subject: [PATCH 23/55] Fix caching problem in refresh_ohlcv closes #5978 --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 19ad4e4b6..5fa852eb0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1294,7 +1294,7 @@ class Exchange: cached_pairs = [] # Gather coroutines to run for pair, timeframe in set(pair_list): - if ((pair, timeframe) not in self._klines + if ((pair, timeframe) not in self._klines or not cache or self._now_is_time_to_refresh(pair, timeframe)): if not since_ms and self.required_candle_call_count > 1: # Multiple calls for one pair - to get more history diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 12b11ff3d..b642b3fa2 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1667,12 +1667,21 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: assert len(res) == len(pairs) assert exchange._api_async.fetch_ohlcv.call_count == 0 + exchange.required_candle_call_count = 1 assert log_has(f"Using cached candle (OHLCV) data for pair {pairs[0][0]}, " f"timeframe {pairs[0][1]} ...", caplog) res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m'), ('XRP/ETH', '1d')], cache=False) assert len(res) == 3 + assert exchange._api_async.fetch_ohlcv.call_count == 3 + + # Test the same again, should NOT return from cache! + exchange._api_async.fetch_ohlcv.reset_mock() + res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m'), ('XRP/ETH', '1d')], + cache=False) + assert len(res) == 3 + assert exchange._api_async.fetch_ohlcv.call_count == 3 @pytest.mark.asyncio From 2b3e7eeb2136e1f50ff9de7573ed1cd82bb23d96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 19:41:36 +0100 Subject: [PATCH 24/55] Use Enum values within bot code --- freqtrade/plugins/pairlist/ShuffleFilter.py | 7 ++++--- tests/plugins/test_pairlist.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 3cefee236..dfdfdda73 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -5,6 +5,7 @@ import logging import random from typing import Any, Dict, List +from freqtrade.enums.runmode import RunMode from freqtrade.plugins.pairlist.IPairList import IPairList @@ -20,12 +21,12 @@ class ShuffleFilter(IPairList): # Apply seed in backtesting mode to get comparable results, # but not in live modes to get a non-repeating order of pairs during live modes. - if config['runmode'].value in ('live', 'dry_run'): + if config.get('runmode') in (RunMode.LIVE, RunMode.DRY_RUN): self._seed = None - logger.info("live mode detected, not applying seed.") + logger.info("Live mode detected, not applying seed.") else: self._seed = pairlistconfig.get('seed') - logger.info("Backtesting mode detected, applying seed value: " + str(self._seed)) + logger.info(f"Backtesting mode detected, applying seed value: {self._seed}") # deprecated since 3.9 #self._random = random.Random(self._seed) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 6333266aa..ba8e6c3c3 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -7,6 +7,7 @@ import pytest import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS +from freqtrade.enums.runmode import RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -657,6 +658,22 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: assert log_has("PerformanceFilter is not available in this mode.", caplog) +def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: + whitelist_conf['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "ShuffleFilter", "seed": 42} + ] + + exchange = get_patched_exchange(mocker, whitelist_conf) + PairListManager(exchange, whitelist_conf) + assert log_has("Backtesting mode detected, applying seed value: 42", caplog) + caplog.clear() + whitelist_conf['runmode'] = RunMode.DRY_RUN + PairListManager(exchange, whitelist_conf) + assert not log_has("Backtesting mode detected, applying seed value: 42", caplog) + assert log_has("Live mode detected, not applying seed.", caplog) + + @pytest.mark.usefixtures("init_persistence") def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee, caplog) -> None: whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC') From 6429205d3920c7c3a7f9c4ce85903ce11b2b4a3f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 19:53:37 +0100 Subject: [PATCH 25/55] Improve Notebook documentation to include Dataprovider fix #5975 --- docs/strategy_analysis_example.md | 4 +++- freqtrade/templates/strategy_analysis_example.ipynb | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index dd7e07824..90d8d8800 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -50,7 +50,9 @@ candles.head() ```python # Load strategy using values set above from freqtrade.resolvers import StrategyResolver +from freqtrade.data.dataprovider import DataProvider strategy = StrategyResolver.load_strategy(config) +strategy.dp = DataProvider(config, None, None) # Generate buy/sell signals using strategy df = strategy.analyze_ticker(candles, {'pair': pair}) @@ -228,7 +230,7 @@ graph = generate_candlestick_graph(pair=pair, # Show graph inline # graph.show() -# Render graph in a separate window +# Render graph in a seperate window graph.show(renderer="browser") ``` diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 99720ae6e..3b937d1c5 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -79,7 +79,9 @@ "source": [ "# Load strategy using values set above\n", "from freqtrade.resolvers import StrategyResolver\n", + "from freqtrade.data.dataprovider import DataProvider\n", "strategy = StrategyResolver.load_strategy(config)\n", + "strategy.dp = DataProvider(config, None, None)\n", "\n", "# Generate buy/sell signals using strategy\n", "df = strategy.analyze_ticker(candles, {'pair': pair})\n", From fd9bf2adb05a79dac902f7223bcfb935583846a6 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 28 Nov 2021 08:23:02 +0900 Subject: [PATCH 26/55] add weekly and monthly to valid keys --- freqtrade/rpc/telegram.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 6c6f745e7..e6624f94d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -112,6 +112,7 @@ class Telegram(RPCHandler): r'/stats$', r'/count$', r'/locks$', r'/balance$', r'/stopbuy$', r'/reload_config$', r'/show_config$', r'/logs$', r'/whitelist$', r'/blacklist$', r'/edge$', + r'/weekly$', r'/weekly \d+$', r'/monthly$', r'/monthly \d+$', r'/forcebuy$', r'/help$', r'/version$'] # Create keys for generation valid_keys_print = [k.replace('$', '') for k in valid_keys] From fb6ae174b97536a589fd857b68699e0518c4662f Mon Sep 17 00:00:00 2001 From: Spat Date: Sun, 28 Nov 2021 11:42:57 +1100 Subject: [PATCH 27/55] Added raw config and retry config to webhook --- docs/webhook-config.md | 33 ++++++++++++++++++++++++-- freqtrade/rpc/webhook.py | 44 +++++++++++++++++++++++++---------- tests/rpc/test_rpc_webhook.py | 11 +++++++++ 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/webhook-config.md b/docs/webhook-config.md index ec944cb50..bea555385 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -50,7 +50,7 @@ Sample configuration (tested using IFTTT). The url in `webhook.url` should point to the correct url for your webhook. If you're using [IFTTT](https://ifttt.com) (as shown in the sample above) please insert your event and key to the url. -You can set the POST body format to Form-Encoded (default) or JSON-Encoded. Use `"format": "form"` or `"format": "json"` respectively. Example configuration for Mattermost Cloud integration: +You can set the POST body format to Form-Encoded (default), JSON-Encoded, or raw data. Use `"format": "form"`, `"format": "json"`, or `"format": "raw"` respectively. Example configuration for Mattermost Cloud integration: ```json "webhook": { @@ -63,7 +63,36 @@ You can set the POST body format to Form-Encoded (default) or JSON-Encoded. Use }, ``` -The result would be POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel. +The result would be a POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel. + +When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be ouput in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example: + +```json + "webhook": { + "enabled": true, + "url": "https://", + "format": "raw", + "webhookstatus": { + "data": "Status: {status}" + } + }, +``` + +The result would be a POST request with e.g. `Status: running` body and `Content-Type: text/plain` header. + +Optional parameters are available to enable automatic retries for webhook messages. The `webhook.retries` parameter can be set for the maximum number of retries the webhook request should attempt if it is unsuccessful (i.e. HTTP response status is not 200). By default this is set to `0` which is disabled. An additional `webhook.retry_delay` parameter can be set to specify the time in seconds between retry attempts. By default this is set to `0.1` (i.e. 100ms). Note that increasing the number of retries or retry delay may slow down the trader if there are connectivity issues with the webhook. Example configuration for retries: + +```json + "webhook": { + "enabled": true, + "url": "https://", + "retries": 3, + "retry_delay": 0.2, + "webhookstatus": { + "status": "Status: {status}" + } + }, +``` Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called. diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index b4c55649e..99077948e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -2,6 +2,7 @@ This module manages webhook communication """ import logging +import time from typing import Any, Dict from requests import RequestException, post @@ -28,12 +29,16 @@ class Webhook(RPCHandler): super().__init__(rpc, config) self._url = self._config['webhook']['url'] - self._format = self._config['webhook'].get('format', 'form') + self._retries = self._config['webhook'].get('retries', 0) + self._retry_delay = self._config['webhook'].get('retry_delay', 0.1) - if self._format != 'form' and self._format != 'json': + if self._retries < 0: self._retries = 0 + if self._retry_delay < 0: self._retry_delay = 0 + + if not (self._format in ['form', 'json', 'raw']): raise NotImplementedError('Unknown webhook format `{}`, possible values are ' - '`form` (default) and `json`'.format(self._format)) + '`form` (default), `json`, and `raw`'.format(self._format)) def cleanup(self) -> None: """ @@ -77,13 +82,28 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" - try: - if self._format == 'form': - post(self._url, data=payload) - elif self._format == 'json': - post(self._url, json=payload) - else: - raise NotImplementedError('Unknown format: {}'.format(self._format)) + success = False + attempts = 0 + while not success and attempts <= self._retries: + if attempts: + if self._retry_delay: time.sleep(self._retry_delay) + logger.info("Retrying webhook...") - except RequestException as exc: - logger.warning("Could not call webhook url. Exception: %s", exc) + attempts += 1 + + try: + if self._format == 'form': + response = post(self._url, data=payload) + elif self._format == 'json': + response = post(self._url, json=payload) + elif self._format == 'raw': + response = post(self._url, data=payload['data'], headers={'Content-Type': 'text/plain'}) + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + + """throw a RequestException if the post was not successful""" + response.raise_for_status() + success = True + + except RequestException as exc: + logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 04e63a3be..735d2ada2 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -292,3 +292,14 @@ def test__send_msg_with_json_format(default_conf, mocker, caplog): webhook._send_msg(msg) assert post.call_args[1] == {'json': msg} + +def test__send_msg_with_raw_format(default_conf, mocker, caplog): + default_conf["webhook"] = get_webhook_dict() + default_conf["webhook"]["format"] = "raw" + webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) + msg = {'data': 'Hello'} + post = MagicMock() + mocker.patch("freqtrade.rpc.webhook.post", post) + webhook._send_msg(msg) + + assert post.call_args[1] == {'data': msg['data'], 'headers': {'Content-Type': 'text/plain'}} From c7d10e2c7e85cabb3b34e8ffd0e3fbc1da4b121c Mon Sep 17 00:00:00 2001 From: incrementby1 <91958753+incrementby1@users.noreply.github.com> Date: Sun, 28 Nov 2021 19:05:02 +0100 Subject: [PATCH 28/55] delete unneeded comment --- freqtrade/plugins/pairlist/ShuffleFilter.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index dfdfdda73..d0eff1b17 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -28,9 +28,6 @@ class ShuffleFilter(IPairList): self._seed = pairlistconfig.get('seed') logger.info(f"Backtesting mode detected, applying seed value: {self._seed}") - # deprecated since 3.9 - #self._random = random.Random(self._seed) - @property def needstickers(self) -> bool: """ From cf5ff9257d432d170af14c4881fc3d098caa817d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 19:39:43 +0100 Subject: [PATCH 29/55] Add plotconfig as property documentation and sample --- docs/plotting.md | 110 +++++++++++++----- freqtrade/templates/base_strategy.py.j2 | 1 + .../subtemplates/plot_config_full.j2 | 30 ++--- 3 files changed, 97 insertions(+), 44 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 9fae38504..b2d7654f6 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -164,16 +164,17 @@ The resulting plot will have the following elements: An advanced plot configuration can be specified in the strategy in the `plot_config` parameter. -Additional features when using plot_config include: +Additional features when using `plot_config` include: * Specify colors per indicator * Specify additional subplots -* Specify indicator pairs to fill area in between +* Specify indicator pairs to fill area in between The sample plot configuration below specifies fixed colors for the indicators. Otherwise, consecutive plots may produce different color schemes each time, making comparisons difficult. It also allows multiple subplots to display both MACD and RSI at the same time. Plot type can be configured using `type` key. Possible types are: + * `scatter` corresponding to `plotly.graph_objects.Scatter` class (default). * `bar` corresponding to `plotly.graph_objects.Bar` class. @@ -182,40 +183,89 @@ Extra parameters to `plotly.graph_objects.*` constructor can be specified in `pl Sample configuration with inline comments explaining the process: ``` python - plot_config = { - 'main_plot': { - # Configuration for main plot indicators. - # Specifies `ema10` to be red, and `ema50` to be a shade of gray - 'ema10': {'color': 'red'}, - 'ema50': {'color': '#CCCCCC'}, - # By omitting color, a random color is selected. - 'sar': {}, - # fill area between senkou_a and senkou_b - 'senkou_a': { - 'color': 'green', #optional - 'fill_to': 'senkou_b', - 'fill_label': 'Ichimoku Cloud', #optional - 'fill_color': 'rgba(255,76,46,0.2)', #optional - }, - # plot senkou_b, too. Not only the area to it. - 'senkou_b': {} +@property +def plot_config(self): + """ + There are a lot of solutions how to build the return dictionary. + The only important point is the return value. + Example: + plot_config = {'main_plot': {}, 'subplots': {}} + + """ + plot_config = {} + plot_config['main_plot'] = { + # Configuration for main plot indicators. + # Assumes 2 parameters, emashort and emalong to be specified. + f'ema_{self.emashort.value}': {'color': 'red'}, + f'ema_{self.emalong.value}': {'color': '#CCCCCC'}, + # By omitting color, a random color is selected. + 'sar': {}, + # fill area between senkou_a and senkou_b + 'senkou_a': { + 'color': 'green', #optional + 'fill_to': 'senkou_b', + 'fill_label': 'Ichimoku Cloud', #optional + 'fill_color': 'rgba(255,76,46,0.2)', #optional }, - 'subplots': { - # Create subplot MACD - "MACD": { - 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, - 'macdsignal': {'color': 'orange'}, - 'macdhist': {'type': 'bar', 'plotly': {'opacity': 0.9}} - }, - # Additional subplot RSI - "RSI": { - 'rsi': {'color': 'red'} - } + # plot senkou_b, too. Not only the area to it. + 'senkou_b': {} + } + plot_config['subplots'] = { + # Create subplot MACD + "MACD": { + 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, + 'macdsignal': {'color': 'orange'}, + 'macdhist': {'type': 'bar', 'plotly': {'opacity': 0.9}} + }, + # Additional subplot RSI + "RSI": { + 'rsi': {'color': 'red'} } } + return plot_config ``` +??? Note "As attribute (former method)" + Assigning plot_config is also possible as Attribute (this used to be the default way). + This has the disadvantage that strategy parameters are not available, preventing certain configurations from working. + + ``` python + plot_config = { + 'main_plot': { + # Configuration for main plot indicators. + # Specifies `ema10` to be red, and `ema50` to be a shade of gray + 'ema10': {'color': 'red'}, + 'ema50': {'color': '#CCCCCC'}, + # By omitting color, a random color is selected. + 'sar': {}, + # fill area between senkou_a and senkou_b + 'senkou_a': { + 'color': 'green', #optional + 'fill_to': 'senkou_b', + 'fill_label': 'Ichimoku Cloud', #optional + 'fill_color': 'rgba(255,76,46,0.2)', #optional + }, + # plot senkou_b, too. Not only the area to it. + 'senkou_b': {} + }, + 'subplots': { + # Create subplot MACD + "MACD": { + 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, + 'macdsignal': {'color': 'orange'}, + 'macdhist': {'type': 'bar', 'plotly': {'opacity': 0.9}} + }, + # Additional subplot RSI + "RSI": { + 'rsi': {'color': 'red'} + } + } + } + + ``` + + !!! Note The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`, `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy. diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 7f5399672..035468d58 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -87,6 +87,7 @@ class {{ strategy }}(IStrategy): 'sell': 'gtc' } {{ plot_config | indent(4) }} + def informative_pairs(self): """ Define additional, informative pair/interval combinations to be cached from the exchange. diff --git a/freqtrade/templates/subtemplates/plot_config_full.j2 b/freqtrade/templates/subtemplates/plot_config_full.j2 index ab02c7892..e3f9e7ca0 100644 --- a/freqtrade/templates/subtemplates/plot_config_full.j2 +++ b/freqtrade/templates/subtemplates/plot_config_full.j2 @@ -1,18 +1,20 @@ -plot_config = { - # Main plot indicators (Moving averages, ...) - 'main_plot': { - 'tema': {}, - 'sar': {'color': 'white'}, - }, - 'subplots': { - # Subplots - each dict defines one additional plot - "MACD": { - 'macd': {'color': 'blue'}, - 'macdsignal': {'color': 'orange'}, +@property +def plot_config(self): + return { + # Main plot indicators (Moving averages, ...) + 'main_plot': { + 'tema': {}, + 'sar': {'color': 'white'}, }, - "RSI": { - 'rsi': {'color': 'red'}, + 'subplots': { + # Subplots - each dict defines one additional plot + "MACD": { + 'macd': {'color': 'blue'}, + 'macdsignal': {'color': 'orange'}, + }, + "RSI": { + 'rsi': {'color': 'red'}, + } } } -} From 0fa5bf54cd92c08a14e7ad2a194f7faa3f30f0ec Mon Sep 17 00:00:00 2001 From: Spat Date: Mon, 29 Nov 2021 10:30:41 +1100 Subject: [PATCH 30/55] Changed comment --- freqtrade/rpc/webhook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 99077948e..f76d50b0e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -101,7 +101,7 @@ class Webhook(RPCHandler): else: raise NotImplementedError('Unknown format: {}'.format(self._format)) - """throw a RequestException if the post was not successful""" + # Throw a RequestException if the post was not successful response.raise_for_status() success = True From 29180a1d2b2d1b02d99d46bcf6904aaa5a62bee5 Mon Sep 17 00:00:00 2001 From: Spat Date: Mon, 29 Nov 2021 10:48:35 +1100 Subject: [PATCH 31/55] Moved retry config to constants --- freqtrade/constants.py | 2 ++ freqtrade/rpc/webhook.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e775e39fc..51ded6c49 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -312,6 +312,8 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'enabled': {'type': 'boolean'}, + 'retries': {'type': 'integer', 'minimum': 0}, + 'retry_delay': {'type': 'number', 'minimum': 0}, 'webhookbuy': {'type': 'object'}, 'webhookbuycancel': {'type': 'object'}, 'webhooksell': {'type': 'object'}, diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index f76d50b0e..1973f212e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -33,9 +33,6 @@ class Webhook(RPCHandler): self._retries = self._config['webhook'].get('retries', 0) self._retry_delay = self._config['webhook'].get('retry_delay', 0.1) - if self._retries < 0: self._retries = 0 - if self._retry_delay < 0: self._retry_delay = 0 - if not (self._format in ['form', 'json', 'raw']): raise NotImplementedError('Unknown webhook format `{}`, possible values are ' '`form` (default), `json`, and `raw`'.format(self._format)) From df09fe5df6740b34e49813e5833898528e7d08d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:20 +0000 Subject: [PATCH 32/55] Bump prompt-toolkit from 3.0.22 to 3.0.23 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.22 to 3.0.23. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.22...3.0.23) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5af330af..4491fecd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,6 @@ psutil==5.8.0 colorama==0.4.4 # Building config files interactively questionary==1.10.0 -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.23 # Extensions to datetime library python-dateutil==2.8.2 From e9e8023d732b3308b1a7b0a3d6e9adc1eed4eaf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:27 +0000 Subject: [PATCH 33/55] Bump ccxt from 1.61.92 to 1.62.42 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.61.92 to 1.62.42. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.61.92...1.62.42) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5af330af..64061ffcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.4 pandas==1.3.4 pandas-ta==0.3.14b -ccxt==1.61.92 +ccxt==1.62.42 # Pin cryptography for now due to rust build errors with piwheels cryptography==36.0.0 aiohttp==3.8.1 From 589c9f55e000f70c71c40617bf0b46619b688d6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:33 +0000 Subject: [PATCH 34/55] Bump scipy from 1.7.2 to 1.7.3 Bumps [scipy](https://github.com/scipy/scipy) from 1.7.2 to 1.7.3. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.7.2...v1.7.3) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index a3da8f0be..05ea21703 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.7.2 +scipy==1.7.3 scikit-learn==1.0.1 scikit-optimize==0.9.0 filelock==3.4.0 From c2a7b1930bfae63e58713f95b6953303c526db4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:36 +0000 Subject: [PATCH 35/55] Bump types-cachetools from 4.2.5 to 4.2.6 Bumps [types-cachetools](https://github.com/python/typeshed) from 4.2.5 to 4.2.6. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4c06e657b..ebad74278 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,7 @@ time-machine==2.4.0 nbconvert==6.3.0 # mypy types -types-cachetools==4.2.5 +types-cachetools==4.2.6 types-filelock==3.2.1 types-requests==2.26.0 types-tabulate==0.8.3 From b0b2fdba708a7493265cd556dd567894bc62fc5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:38 +0000 Subject: [PATCH 36/55] Bump aiofiles from 0.7.0 to 0.8.0 Bumps [aiofiles](https://github.com/Tinche/aiofiles) from 0.7.0 to 0.8.0. - [Release notes](https://github.com/Tinche/aiofiles/releases) - [Commits](https://github.com/Tinche/aiofiles/compare/v0.7.0...v0.8.0) --- updated-dependencies: - dependency-name: aiofiles dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5af330af..828b5754e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ sdnotify==0.3.2 fastapi==0.70.0 uvicorn==0.15.0 pyjwt==2.3.0 -aiofiles==0.7.0 +aiofiles==0.8.0 psutil==5.8.0 # Support for colorized terminal output From 39c3175b69ae65292d88a8cca1f3cb9371b084ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:43 +0000 Subject: [PATCH 37/55] Bump types-python-dateutil from 2.8.2 to 2.8.3 Bumps [types-python-dateutil](https://github.com/python/typeshed) from 2.8.2 to 2.8.3. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4c06e657b..a6f066be9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,4 +26,4 @@ types-requests==2.26.0 types-tabulate==0.8.3 # Extensions to datetime library -types-python-dateutil==2.8.2 \ No newline at end of file +types-python-dateutil==2.8.3 \ No newline at end of file From b81d768eb3b355edbfe574cb837971f80fbeb97a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:45 +0000 Subject: [PATCH 38/55] Bump types-requests from 2.26.0 to 2.26.1 Bumps [types-requests](https://github.com/python/typeshed) from 2.26.0 to 2.26.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4c06e657b..34cb222d4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ nbconvert==6.3.0 # mypy types types-cachetools==4.2.5 types-filelock==3.2.1 -types-requests==2.26.0 +types-requests==2.26.1 types-tabulate==0.8.3 # Extensions to datetime library From 24997fb36ff7921dcb5f6acc4f1b33f37707fb64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 03:01:50 +0000 Subject: [PATCH 39/55] Bump mkdocs-material from 7.3.6 to 8.0.1 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.3.6 to 8.0.1. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Upgrade guide](https://github.com/squidfunk/mkdocs-material/blob/master/docs/upgrade.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.3.6...8.0.1) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 772919436..351f45af6 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.3 -mkdocs-material==7.3.6 +mkdocs-material==8.0.1 mdx_truly_sane_lists==1.2 pymdown-extensions==9.1 From c126d2530aa8dc10f814e984275b061495d84aef Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 29 Nov 2021 14:32:33 +0900 Subject: [PATCH 40/55] Add few sentences on docs - Add warning that PrecisionFilter can't be used on backtest that use multiple strategies - Add note that not all pairlist handlers can be used on backtest --- docs/backtesting.md | 2 +- docs/includes/pairlists.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 49a94b05e..a49e4700a 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -115,7 +115,7 @@ The result of backtesting will confirm if your bot has better odds of making a p All profit calculations include fees, and freqtrade will use the exchange's default fees for the calculation. !!! Warning "Using dynamic pairlists for backtesting" - Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist. + Using dynamic pairlists is possible (not all of the handlers are allowed to be used in backtest mode), however it relies on the current market conditions - which will not reflect the historic status of the pairlist. Also, when using pairlists other than StaticPairlist, reproducibility of backtesting-results cannot be guaranteed. Please read the [pairlists documentation](plugins.md#pairlists) for more information. diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index bbfe74510..29e20a32f 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -220,6 +220,9 @@ As this Filter uses past performance of the bot, it'll have some startup-period Filters low-value coins which would not allow setting stoplosses. +!!! Warning "Backtesting" + `PrecisionFilter` does not support backtesting mode using multiple strategies. + #### PriceFilter The `PriceFilter` allows filtering of pairs by price. Currently the following price filters are supported: From 57e55eb93836d46c2e95f66f97fb2ec7c6bcc7af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 06:00:32 +0000 Subject: [PATCH 41/55] Bump time-machine from 2.4.0 to 2.4.1 Bumps [time-machine](https://github.com/adamchainz/time-machine) from 2.4.0 to 2.4.1. - [Release notes](https://github.com/adamchainz/time-machine/releases) - [Changelog](https://github.com/adamchainz/time-machine/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/time-machine/compare/2.4.0...2.4.1) --- updated-dependencies: - dependency-name: time-machine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d127eabcc..055a2a35d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,7 +14,7 @@ pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.4.0 +time-machine==2.4.1 # Convert jupyter notebooks to markdown documents nbconvert==6.3.0 From 018407852a1cca078e0cbdb6c5b31f1f5eb914e2 Mon Sep 17 00:00:00 2001 From: Spat Date: Mon, 29 Nov 2021 18:17:59 +1100 Subject: [PATCH 42/55] Added missing webhook config params to constants --- freqtrade/constants.py | 6 ++++++ freqtrade/rpc/webhook.py | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 51ded6c49..e074718ca 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -50,6 +50,8 @@ USERPATH_STRATEGIES = 'strategies' USERPATH_NOTEBOOKS = 'notebooks' TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] +WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw'] + ENV_VAR_PREFIX = 'FREQTRADE__' NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired') @@ -312,12 +314,16 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'enabled': {'type': 'boolean'}, + 'url': {'type': 'string'}, + 'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'}, 'retries': {'type': 'integer', 'minimum': 0}, 'retry_delay': {'type': 'number', 'minimum': 0}, 'webhookbuy': {'type': 'object'}, 'webhookbuycancel': {'type': 'object'}, + 'webhookbuyfill': {'type': 'object'}, 'webhooksell': {'type': 'object'}, 'webhooksellcancel': {'type': 'object'}, + 'webhooksellfill': {'type': 'object'}, 'webhookstatus': {'type': 'object'}, }, }, diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 1973f212e..2a848787d 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -33,10 +33,6 @@ class Webhook(RPCHandler): self._retries = self._config['webhook'].get('retries', 0) self._retry_delay = self._config['webhook'].get('retry_delay', 0.1) - if not (self._format in ['form', 'json', 'raw']): - raise NotImplementedError('Unknown webhook format `{}`, possible values are ' - '`form` (default), `json`, and `raw`'.format(self._format)) - def cleanup(self) -> None: """ Cleanup pending module resources. From 2e5147745540686d04821d33063b5b186b564b4b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Nov 2021 19:32:16 +0100 Subject: [PATCH 43/55] Update mkdocs file to 8.0 --- mkdocs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index b9c053324..9eebd75e3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,8 +81,9 @@ markdown_extensions: - pymdownx.snippets: base_path: docs check_paths: true - - pymdownx.tabbed - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true - pymdownx.tasklist: custom_checkbox: true - mdx_truly_sane_lists From f8cb3d290188269808d6f3e88bd172fb526b88d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Nov 2021 19:52:40 +0100 Subject: [PATCH 44/55] Restore openAPI functioning --- freqtrade/rpc/api_server/api_schemas.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index d0e772848..98cc8e15a 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -132,13 +132,10 @@ class UnfilledTimeout(BaseModel): exit_timeout_count: Optional[int] -class OrderTypeValues(Enum): +class OrderTypeValues(str, Enum): limit = 'limit' market = 'market' - class Config: - use_enum_values = True - class OrderTypes(BaseModel): buy: OrderTypeValues From dfb148f8d7cdbe6f3a5f8ca345f54f74943538dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Nov 2021 19:54:54 +0100 Subject: [PATCH 45/55] Fix formatting --- freqtrade/rpc/webhook.py | 8 +++++--- tests/rpc/test_rpc_webhook.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 2a848787d..58b75769e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -79,7 +79,8 @@ class Webhook(RPCHandler): attempts = 0 while not success and attempts <= self._retries: if attempts: - if self._retry_delay: time.sleep(self._retry_delay) + if self._retry_delay: + time.sleep(self._retry_delay) logger.info("Retrying webhook...") attempts += 1 @@ -90,10 +91,11 @@ class Webhook(RPCHandler): elif self._format == 'json': response = post(self._url, json=payload) elif self._format == 'raw': - response = post(self._url, data=payload['data'], headers={'Content-Type': 'text/plain'}) + response = post(self._url, data=payload['data'], + headers={'Content-Type': 'text/plain'}) else: raise NotImplementedError('Unknown format: {}'.format(self._format)) - + # Throw a RequestException if the post was not successful response.raise_for_status() success = True diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 735d2ada2..17d1baca9 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -293,6 +293,7 @@ def test__send_msg_with_json_format(default_conf, mocker, caplog): assert post.call_args[1] == {'json': msg} + def test__send_msg_with_raw_format(default_conf, mocker, caplog): default_conf["webhook"] = get_webhook_dict() default_conf["webhook"]["format"] = "raw" From 60eca8b1f1de636d19b5e6d2c07f412c8c25e866 Mon Sep 17 00:00:00 2001 From: incrementby1 <91958753+incrementby1@users.noreply.github.com> Date: Mon, 29 Nov 2021 20:35:43 +0100 Subject: [PATCH 46/55] revert to random object --- freqtrade/plugins/pairlist/ShuffleFilter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index d0eff1b17..55cf9938f 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -28,6 +28,8 @@ class ShuffleFilter(IPairList): self._seed = pairlistconfig.get('seed') logger.info(f"Backtesting mode detected, applying seed value: {self._seed}") + self._random = random.Random(self._seed) + @property def needstickers(self) -> bool: """ @@ -53,7 +55,6 @@ class ShuffleFilter(IPairList): :return: new whitelist """ # Shuffle is done inplace - random.seed(self._seed) - random.shuffle(pairlist) + self._random.shuffle(pairlist) return pairlist From 85b1f6f6b34491ae88026789b3e765eba76a0b75 Mon Sep 17 00:00:00 2001 From: incrementby1 <91958753+incrementby1@users.noreply.github.com> Date: Mon, 29 Nov 2021 20:44:51 +0100 Subject: [PATCH 47/55] Update pairlists.md --- docs/includes/pairlists.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index c6b320e62..9ef4204f4 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -254,7 +254,7 @@ Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - #### ShuffleFilter -Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. +Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. !!! Tip You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. ShuffleFilter will automatically detect runmodes and apply the `seed` only for backtesting modes - if a `seed` value is set. From 231b1e2f572c2017eee89cacdea1e126063d1065 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Nov 2021 06:58:32 +0100 Subject: [PATCH 48/55] Improve Async error message content --- freqtrade/exchange/exchange.py | 4 ++-- tests/exchange/test_exchange.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 5fa852eb0..e25be9ae1 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1263,7 +1263,7 @@ class Exchange: results = await asyncio.gather(*input_coro, return_exceptions=True) for res in results: if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) + logger.warning(f"Async code raised an exception: {repr(res)}") if raise_: raise continue @@ -1324,7 +1324,7 @@ class Exchange: # handle caching for res in results: if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) + logger.warning(f"Async code raised an exception: {repr(res)}") continue # Deconstruct tuple (has 3 elements) pair, timeframe, ticks = res diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b642b3fa2..5a35675a8 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1777,7 +1777,7 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): assert len(res) == 1 # Test that each is in list at least once as order is not guaranteed assert log_has("Error loading ETH/BTC. Result was [[]].", caplog) - assert log_has("Async code raised an exception: TypeError", caplog) + assert log_has("Async code raised an exception: TypeError()", caplog) def test_get_next_limit_in_list(): From f0abe218a2cbdcd0a7296b07894d23a5233ed39e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Nov 2021 07:02:49 +0100 Subject: [PATCH 49/55] Batch ohlcv requests to not overwelm ccxt's async throttler closes #6003 --- freqtrade/exchange/exchange.py | 43 ++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e25be9ae1..0ae78cf1b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1317,27 +1317,30 @@ class Exchange: ) cached_pairs.append((pair, timeframe)) - results = asyncio.get_event_loop().run_until_complete( - asyncio.gather(*input_coroutines, return_exceptions=True)) - results_df = {} - # handle caching - for res in results: - if isinstance(res, Exception): - logger.warning(f"Async code raised an exception: {repr(res)}") - continue - # Deconstruct tuple (has 3 elements) - pair, timeframe, ticks = res - # keeping last candle time as last refreshed time of the pair - if ticks: - self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 - # keeping parsed dataframe in cache - ohlcv_df = ohlcv_to_dataframe( - ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) - results_df[(pair, timeframe)] = ohlcv_df - if cache: - self._klines[(pair, timeframe)] = ohlcv_df + # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling + for input_coro in chunks(input_coroutines, 100): + results = asyncio.get_event_loop().run_until_complete( + asyncio.gather(*input_coro, return_exceptions=True)) + + # handle caching + for res in results: + if isinstance(res, Exception): + logger.warning(f"Async code raised an exception: {repr(res)}") + continue + # Deconstruct tuple (has 3 elements) + pair, timeframe, ticks = res + # keeping last candle time as last refreshed time of the pair + if ticks: + self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 + # keeping parsed dataframe in cache + ohlcv_df = ohlcv_to_dataframe( + ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + results_df[(pair, timeframe)] = ohlcv_df + if cache: + self._klines[(pair, timeframe)] = ohlcv_df + # Return cached klines for pair, timeframe in cached_pairs: results_df[(pair, timeframe)] = self.klines((pair, timeframe), copy=False) From 542963c7a60e699c1ebd0602ecd4b5be7a86f6a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Nov 2021 19:45:20 +0100 Subject: [PATCH 50/55] Reduce code complexity by combining buy and buy_fill methods --- docs/webhook-config.md | 5 ++++- freqtrade/freqtradebot.py | 26 ++++++-------------------- mkdocs.yml | 1 + 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/docs/webhook-config.md b/docs/webhook-config.md index bea555385..40915c988 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -104,7 +104,8 @@ Possible parameters are: * `trade_id` * `exchange` * `pair` -* `limit` +* ~~`limit` # Deprecated - should no longer be used.~~ +* `open_rate` * `amount` * `open_date` * `stake_amount` @@ -146,6 +147,8 @@ Possible parameters are: * `stake_amount` * `stake_currency` * `fiat_currency` +* `order_type` +* `current_rate` * `buy_tag` ### Webhooksell diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a6d1b36b9..32f08c178 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -592,17 +592,19 @@ class FreqtradeBot(LoggingMixin): return True - def _notify_enter(self, trade: Trade, order_type: str) -> None: + def _notify_enter(self, trade: Trade, order_type: Optional[str] = None, + fill: bool = False) -> None: """ Sends rpc notification when a buy occurred. """ msg = { 'trade_id': trade.id, - 'type': RPCMessageType.BUY, + 'type': RPCMessageType.BUY_FILL if fill else RPCMessageType.BUY, 'buy_tag': trade.buy_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, - 'limit': trade.open_rate, + 'limit': trade.open_rate, # Deprecated (?) + 'open_rate': trade.open_rate, 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], @@ -641,22 +643,6 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_enter_fill(self, trade: Trade) -> None: - msg = { - 'trade_id': trade.id, - 'type': RPCMessageType.BUY_FILL, - 'buy_tag': trade.buy_tag, - 'exchange': self.exchange.name.capitalize(), - 'pair': trade.pair, - 'open_rate': trade.open_rate, - 'stake_amount': trade.stake_amount, - 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency', None), - 'amount': trade.amount, - 'open_date': trade.open_date, - } - self.rpc.send_msg(msg) - # # SELL / exit positions / close trades logic and methods # @@ -1312,7 +1298,7 @@ class FreqtradeBot(LoggingMixin): self.wallets.update() elif not trade.open_order_id: # Buy fill - self._notify_enter_fill(trade) + self._notify_enter(trade, fill=True) return False diff --git a/mkdocs.yml b/mkdocs.yml index 9eebd75e3..fb1b80ebf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -86,4 +86,5 @@ markdown_extensions: alternate_style: true - pymdownx.tasklist: custom_checkbox: true + - pymdownx.tilde - mdx_truly_sane_lists From a2a974fc6dc4a900e17363c5405d31d481b74164 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Nov 2021 20:32:34 +0100 Subject: [PATCH 51/55] correctly apply leverage to backtesting --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8ada54288..36705b0e2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -490,7 +490,7 @@ class Backtesting: open_rate=row[OPEN_IDX], open_date=current_time, stake_amount=stake_amount, - amount=round(stake_amount / row[OPEN_IDX], 8), + amount=round((stake_amount / row[OPEN_IDX]) * leverage, 8), fee_open=self.fee, fee_close=self.fee, is_open=True, From 8b2fbb64325c996886e0a2940af16da98b5cf16e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Nov 2021 20:42:18 +0100 Subject: [PATCH 52/55] Add leveraged backtest detail test --- tests/optimize/__init__.py | 1 + tests/optimize/test_backtest_detail.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 05c55456c..ce6f17f6e 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -36,6 +36,7 @@ class BTContainer(NamedTuple): trailing_stop_positive_offset: float = 0.0 use_sell_signal: bool = False use_custom_stoploss: bool = False + leverage: float = 1.0 def _get_frame_time_from_offset(offset): diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 6db88d123..798fdc302 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -536,6 +536,23 @@ tc33 = BTContainer(data=[ )] ) +# Test 34: (copy of test25 with leverage) +# Sell with signal sell in candle 3 (stoploss also triggers on this candle) +# Stoploss at 1%. +# Sell-signal wins over stoploss +tc34 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4986, 6172, 0, 0], + [3, 5010, 5010, 4986, 5010, 6172, 0, 1], + [4, 5010, 5010, 4855, 4995, 6172, 0, 0], # Triggers stoploss + sellsignal acted on + [5, 4995, 4995, 4950, 4950, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 1}, profit_perc=0.002 * 5.0, use_sell_signal=True, + leverage=5.0, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=4)] +) + TESTS = [ tc0, tc1, @@ -571,6 +588,7 @@ TESTS = [ tc31, tc32, tc33, + tc34, # TODO-lev: Add tests for short here ] @@ -593,14 +611,19 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: mocker.patch("freqtrade.exchange.Exchange.get_fee", return_value=0.0) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) + mocker.patch("freqtrade.exchange.Binance.get_max_leverage", return_value=100) patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) backtesting.required_startup = 0 + if data.leverage > 1.0: + # TODO-lev: Should we initialize this properly?? + backtesting._can_short = True backtesting.strategy.advise_entry = lambda a, m: frame backtesting.strategy.advise_exit = lambda a, m: frame backtesting.strategy.use_custom_stoploss = data.use_custom_stoploss + backtesting.strategy.leverage = lambda **kwargs: data.leverage caplog.set_level(logging.DEBUG) pair = "UNITTEST/BTC" From c22f381dfe69a2a9e3b5fabc3318c80d4c06153d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Nov 2021 20:46:47 +0100 Subject: [PATCH 53/55] Fix Schema issue closes #6010 --- freqtrade/rpc/api_server/api_schemas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 98cc8e15a..c1720a836 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -126,9 +126,9 @@ class Daily(BaseModel): class UnfilledTimeout(BaseModel): - buy: int - sell: int - unit: str + buy: Optional[int] + sell: Optional[int] + unit: Optional[str] exit_timeout_count: Optional[int] From 5ce1eeecf5b7cdd892191c599ac257f7705cf70e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Nov 2021 20:19:59 +0100 Subject: [PATCH 54/55] Reorder messages to be sent in correct order buy first, then buy fill, sell first, then sell fill. --- freqtrade/freqtradebot.py | 30 ++++++++++++++++-------------- tests/rpc/test_rpc_telegram.py | 6 +++--- tests/test_freqtradebot.py | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 32f08c178..7d8e0ec2f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -278,7 +278,8 @@ class FreqtradeBot(LoggingMixin): if order: logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.") self.update_trade_state(trade, order.order_id, - stoploss_order=order.ft_order_side == 'stoploss') + stoploss_order=order.ft_order_side == 'stoploss', + send_msg=False) trades: List[Trade] = Trade.get_open_trades_without_assigned_fees() for trade in trades: @@ -286,7 +287,7 @@ class FreqtradeBot(LoggingMixin): order = trade.select_order('buy', False) if order: logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") - self.update_trade_state(trade, order.order_id) + self.update_trade_state(trade, order.order_id, send_msg=False) def handle_insufficient_funds(self, trade: Trade): """ @@ -308,7 +309,7 @@ class FreqtradeBot(LoggingMixin): order = trade.select_order('buy', False) if order: logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") - self.update_trade_state(trade, order.order_id) + self.update_trade_state(trade, order.order_id, send_msg=False) def refind_lost_order(self, trade): """ @@ -578,10 +579,6 @@ class FreqtradeBot(LoggingMixin): ) trade.orders.append(order_obj) - # Update fees if order is closed - if order_status == 'closed': - self.update_trade_state(trade, order_id, order) - Trade.query.session.add(trade) Trade.commit() @@ -590,6 +587,10 @@ class FreqtradeBot(LoggingMixin): self._notify_enter(trade, order_type) + # Update fees if order is closed + if order_status == 'closed': + self.update_trade_state(trade, order_id, order) + return True def _notify_enter(self, trade: Trade, order_type: Optional[str] = None, @@ -1140,16 +1141,16 @@ class FreqtradeBot(LoggingMixin): trade.sell_order_status = '' trade.close_rate_requested = limit trade.sell_reason = exit_tag or sell_reason.sell_reason - # In case of market sell orders the order can be closed immediately - if order.get('status', 'unknown') in ('closed', 'expired'): - self.update_trade_state(trade, trade.open_order_id, order) - Trade.commit() # Lock pair for one candle to prevent immediate re-buys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') self._notify_exit(trade, order_type) + # In case of market sell orders the order can be closed immediately + if order.get('status', 'unknown') in ('closed', 'expired'): + self.update_trade_state(trade, trade.open_order_id, order) + Trade.commit() return True @@ -1246,13 +1247,14 @@ class FreqtradeBot(LoggingMixin): # def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None, - stoploss_order: bool = False) -> bool: + stoploss_order: bool = False, send_msg: bool = True) -> bool: """ Checks trades with open orders and updates the amount if necessary Handles closing both buy and sell orders. :param trade: Trade object of the trade we're analyzing :param order_id: Order-id of the order we're analyzing :param action_order: Already acquired order object + :param send_msg: Send notification - should always be True except in "recovery" methods :return: True if order has been cancelled without being filled partially, False otherwise """ if not order_id: @@ -1292,11 +1294,11 @@ class FreqtradeBot(LoggingMixin): # Updating wallets when order is closed if not trade.is_open: - if not stoploss_order and not trade.open_order_id: + if send_msg and not stoploss_order and not trade.open_order_id: self._notify_exit(trade, '', True) self.handle_protections(trade.pair) self.wallets.update() - elif not trade.open_order_id: + elif send_msg and not trade.open_order_id: # Buy fill self._notify_enter(trade, fill=True) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 6c32e59fc..6adce7b4d 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -937,7 +937,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, telegram._forcesell(update=update, context=context) assert msg_mock.call_count == 4 - last_msg = msg_mock.call_args_list[-1][0][0] + last_msg = msg_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -1001,7 +1001,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, assert msg_mock.call_count == 4 - last_msg = msg_mock.call_args_list[-1][0][0] + last_msg = msg_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -1055,7 +1055,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None # Called for each trade 2 times assert msg_mock.call_count == 8 - msg = msg_mock.call_args_list[1][0][0] + msg = msg_mock.call_args_list[0][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e5dae5461..dd1fcd6e2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2979,7 +2979,7 @@ def test_execute_trade_exit_market_order(default_conf_usdt, ticker_usdt, fee, assert trade.close_profit == 0.09451372 assert rpc_mock.call_count == 3 - last_msg = rpc_mock.call_args_list[-1][0][0] + last_msg = rpc_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, From 0375a083029ff1ba2a88944b0dd8d79b929651e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Dec 2021 20:32:23 +0100 Subject: [PATCH 55/55] use to_hdf instead of HDFStore --- freqtrade/data/history/hdf5datahandler.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index dd60530aa..1ede3de98 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -61,10 +61,10 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) - ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc') - ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date']) - - ds.close() + _data.loc[:, self._columns].to_hdf( + filename, key, mode='a', complevel=9, complib='blosc', + format='table', data_columns=['date'] + ) def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame: @@ -142,11 +142,11 @@ class HDF5DataHandler(IDataHandler): """ key = self._pair_trades_key(pair) - ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair), - mode='a', complevel=9, complib='blosc') - ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS), - format='table', data_columns=['timestamp']) - ds.close() + pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf( + self._pair_trades_filename(self._datadir, pair), key, + mode='a', complevel=9, complib='blosc', + format='table', data_columns=['timestamp'] + ) def trades_append(self, pair: str, data: TradeList): """