From 2b61aa282a434f4011fbd5e5720270b4f6eefe35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Wed, 18 May 2022 03:41:10 +0530 Subject: [PATCH 001/162] Removed None in dict.get() https://stackoverflow.com/a/12631641 Extra Changes: freqtrade\freqtradebot.py:70 freqtrade\plugins\pairlistmanager.py:31 --- freqtrade/commands/hyperopt_commands.py | 2 +- freqtrade/configuration/configuration.py | 8 ++++---- freqtrade/exchange/exchange.py | 12 ++++++------ freqtrade/freqtradebot.py | 14 +++++++------- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/optimize/hyperopt_tools.py | 16 ++++++++-------- freqtrade/plugins/pairlist/AgeFilter.py | 2 +- freqtrade/plugins/pairlist/PerformanceFilter.py | 2 +- freqtrade/plugins/pairlist/PriceFilter.py | 2 +- .../plugins/pairlist/rangestabilityfilter.py | 2 +- freqtrade/plugins/pairlistmanager.py | 2 +- freqtrade/rpc/api_server/api_v1.py | 2 +- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/rpc/telegram.py | 6 +++--- freqtrade/rpc/webhook.py | 14 +++++++------- freqtrade/wallets.py | 6 +++--- 17 files changed, 50 insertions(+), 50 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 344828282..19e291ea7 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -24,7 +24,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: print_colorized = config.get('print_colorized', False) print_json = config.get('print_json', False) - export_csv = config.get('export_csv', None) + export_csv = config.get('export_csv') no_details = config.get('hyperopt_list_no_details', False) no_header = False diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 96b585cd1..e4c7b8f9d 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -127,7 +127,7 @@ class Configuration: # Default to in-memory db for dry_run if not specified config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL else: - if not config.get('db_url', None): + if not config.get('db_url'): config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') @@ -180,7 +180,7 @@ class Configuration: config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False) logger.info('Using user-data directory: %s ...', config['user_data_dir']) - config.update({'datadir': create_datadir(config, self.args.get('datadir', None))}) + config.update({'datadir': create_datadir(config, self.args.get('datadir'))}) logger.info('Using data directory: %s ...', config.get('datadir')) if self.args.get('exportfilename'): @@ -219,7 +219,7 @@ class Configuration: if config.get('max_open_trades') == -1: config['max_open_trades'] = float('inf') - if self.args.get('stake_amount', None): + if self.args.get('stake_amount'): # Convert explicitly to float to support CLI argument for both unlimited and value try: self.args['stake_amount'] = float(self.args['stake_amount']) @@ -459,7 +459,7 @@ class Configuration: configuration instead of the content) """ if (argname in self.args and self.args[argname] is not None - and self.args[argname] is not False): + and self.args[argname] is not False): config.update({argname: self.args[argname]}) if logfun: diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d2766cd6d..685f86899 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -383,11 +383,11 @@ class Exchange: Ensures that Configured mode aligns to """ return ( - market.get('quote', None) is not None - and market.get('base', None) is not None + market.get('quote') is not None + and market.get('base') is not None and (self.precisionMode != TICK_SIZE # Too low precision will falsify calculations - or market.get('precision', {}).get('price', None) > 1e-11) + or market.get('precision', {}).get('price') > 1e-11) and ((self.trading_mode == TradingMode.SPOT and self.market_is_spot(market)) or (self.trading_mode == TradingMode.MARGIN and self.market_is_margin(market)) or (self.trading_mode == TradingMode.FUTURES and self.market_is_future(market))) @@ -537,7 +537,7 @@ class Exchange: # The internal info array is different for each particular market, # its contents depend on the exchange. # It can also be a string or similar ... so we need to verify that first. - elif (isinstance(self.markets[pair].get('info', None), dict) + elif (isinstance(self.markets[pair].get('info'), dict) and self.markets[pair].get('info', {}).get('prohibitedIn', False)): # Warn users about restricted pairs in whitelist. # We cannot determine reliably if Users are affected. @@ -1593,7 +1593,7 @@ class Exchange: def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, price: float = 1, taker_or_maker: str = 'maker') -> float: try: - if self._config['dry_run'] and self._config.get('fee', None) is not None: + if self._config['dry_run'] and self._config.get('fee') is not None: return self._config['fee'] # validate that markets are loaded before trying to get fee if self._api.markets is None or len(self._api.markets) == 0: @@ -1654,7 +1654,7 @@ class Exchange: fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') except ExchangeError: - fee_to_quote_rate = self._config['exchange'].get('unknown_fee_rate', None) + fee_to_quote_rate = self._config['exchange'].get('unknown_fee_rate') if not fee_to_quote_rate: return None return round((self._contracts_to_amount( diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 315db3ae6..cbd44a755 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -67,7 +67,7 @@ class FreqtradeBot(LoggingMixin): self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - init_db(self.config.get('db_url', None), clean_open_orders=self.config['dry_run']) + init_db(self.config['db_url'], clean_open_orders=self.config['dry_run']) self.wallets = Wallets(self.config, self.exchange) @@ -638,7 +638,7 @@ class FreqtradeBot(LoggingMixin): ) order_obj = Order.parse_from_ccxt_object(order, pair, side) order_id = order['id'] - order_status = order.get('status', None) + order_status = order.get('status') logger.info(f"Order #{order_id} was created for {pair} and status is {order_status}.") # we assume the order is executed at the price requested @@ -845,7 +845,7 @@ class FreqtradeBot(LoggingMixin): 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency', None), + 'fiat_currency': self.config.get('fiat_display_currency'), 'amount': safe_value_fallback(order, 'filled', 'amount') or trade.amount, 'open_date': trade.open_date or datetime.utcnow(), 'current_rate': current_rate, @@ -874,7 +874,7 @@ class FreqtradeBot(LoggingMixin): 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency', None), + 'fiat_currency': self.config.get('fiat_display_currency'), 'amount': trade.amount, 'open_date': trade.open_date, 'current_rate': current_rate, @@ -1529,7 +1529,7 @@ class FreqtradeBot(LoggingMixin): 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.utcnow(), 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency', None), + 'fiat_currency': self.config.get('fiat_display_currency'), } if 'fiat_display_currency' in self.config: @@ -1578,7 +1578,7 @@ class FreqtradeBot(LoggingMixin): 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.now(timezone.utc), 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency', None), + 'fiat_currency': self.config.get('fiat_display_currency'), 'reason': reason, } @@ -1640,7 +1640,7 @@ class FreqtradeBot(LoggingMixin): if order['status'] in constants.NON_OPEN_EXCHANGE_STATES: # If a entry order was closed, force update on stoploss on exchange - if order.get('side', None) == trade.entry_side: + if order.get('side') == trade.entry_side: trade = self.cancel_stoploss_on_exchange(trade) # TODO: Margin will need to use interest_rate as well. # interest_rate = self.exchange.get_interest_rate() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4e604898f..f1ccd3b51 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -87,7 +87,7 @@ class Backtesting: self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config) self.dataprovider = DataProvider(self.config, self.exchange) - if self.config.get('strategy_list', None): + if self.config.get('strategy_list'): for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat @@ -123,7 +123,7 @@ class Backtesting: if len(self.pairlists.whitelist) == 0: raise OperationalException("No pair in whitelist.") - if config.get('fee', None) is not None: + if config.get('fee') is not None: self.fee = config['fee'] else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 1dafb483c..a267ca4ac 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -455,7 +455,7 @@ class Hyperopt: return self.opt.ask(n_points=n_points), [False for _ in range(n_points)] def start(self) -> None: - self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) + self.random_state = self._set_random_state(self.config.get('hyperopt_random_state')) logger.info(f"Using optimizer random state: {self.random_state}") self.hyperopt_table_header = -1 # Initialize spaces ... diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 0421e6e38..ab6ef013b 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -127,14 +127,14 @@ class HyperoptTools(): 'only_profitable': config.get('hyperopt_list_profitable', False), 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), - 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), - 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), - 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), - 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), - 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None), - 'filter_min_objective': config.get('hyperopt_list_min_objective', None), - 'filter_max_objective': config.get('hyperopt_list_max_objective', None), + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time'), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time'), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit'), + 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit'), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit'), + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit'), + 'filter_min_objective': config.get('hyperopt_list_min_objective'), + 'filter_max_objective': config.get('hyperopt_list_max_objective'), } if not HyperoptTools._test_hyperopt_results_exist(results_file): # No file found. diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 418c0f14e..786f32e88 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -30,7 +30,7 @@ class AgeFilter(IPairList): self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) - self._max_days_listed = pairlistconfig.get('max_days_listed', None) + self._max_days_listed = pairlistconfig.get('max_days_listed') candle_limit = exchange.ohlcv_candle_limit('1d', self._config['candle_type_def']) if self._min_days_listed < 1: diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 5b02a47ab..8e0b407c3 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -21,7 +21,7 @@ class PerformanceFilter(IPairList): super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) self._minutes = pairlistconfig.get('minutes', 0) - self._min_profit = pairlistconfig.get('min_profit', None) + self._min_profit = pairlistconfig.get('min_profit') @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index 009789eaf..4c5db52b1 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -70,7 +70,7 @@ class PriceFilter(IPairList): :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ - if ticker.get('last', None) is None or ticker.get('last') == 0: + if ticker.get('last') is None or ticker.get('last') == 0: self.log_once(f"Removed {pair} from whitelist, because " "ticker['last'] is empty (Usually no trade in the last 24h).", logger.info) diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index de016c3a6..f3e7bc0d6 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -27,7 +27,7 @@ class RangeStabilityFilter(IPairList): self._days = pairlistconfig.get('lookback_days', 10) self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) - self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None) + self._max_rate_of_change = pairlistconfig.get('max_rate_of_change') self._refresh_period = pairlistconfig.get('refresh_period', 1440) self._def_candletype = self._config['candle_type_def'] diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 2ae67a157..3ddad4a5e 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -28,7 +28,7 @@ class PairListManager(LoggingMixin): self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False - for pairlist_handler_config in self._config.get('pairlists', None): + for pairlist_handler_config in self._config.get('pairlists', []): pairlist_handler = PairListResolver.load_pairlist( pairlist_handler_config['method'], exchange=exchange, diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index a8b9873d7..c8299a153 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -281,7 +281,7 @@ def get_strategy(strategy: str, config=Depends(get_config)): def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Optional[str] = None, candletype: Optional[CandleType] = None, config=Depends(get_config)): - dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) + dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv')) trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) pair_interval = dh.ohlcv_get_available_data(config['datadir'], trading_mode) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a98e3f96d..4c03b8bd4 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -96,7 +96,7 @@ class RPC: """ self._freqtrade = freqtrade self._config: Dict[str, Any] = freqtrade.config - if self._config.get('fiat_display_currency', None): + if self._config.get('fiat_display_currency'): self._fiat_converter = CryptoToFiatConverter() @staticmethod @@ -600,7 +600,7 @@ class RPC: else: try: pair = self._freqtrade.exchange.get_valid_pair_combination(coin, stake_currency) - rate = tickers.get(pair, {}).get('last', None) + rate = tickers.get(pair, {}).get('last') if rate: if pair.startswith(stake_currency) and not pair.endswith(stake_currency): rate = 1.0 / rate diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f26de8b5c..81fb206cc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -241,7 +241,7 @@ class Telegram(RPCHandler): f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}" f" (#{msg['trade_id']})\n" ) - message += f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag', None) else "" + message += f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag') else "" message += f"*Amount:* `{msg['amount']:.8f}`\n" if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0: message += f"*Leverage:* `{msg['leverage']}`\n" @@ -254,7 +254,7 @@ class Telegram(RPCHandler): message += f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}" - if msg.get('fiat_currency', None): + if msg.get('fiat_currency'): message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}" message += ")`" @@ -270,7 +270,7 @@ class Telegram(RPCHandler): msg['enter_tag'] = msg['enter_tag'] if "enter_tag" in msg.keys() else None msg['emoji'] = self._get_sell_emoji(msg) msg['leverage_text'] = (f"*Leverage:* `{msg['leverage']:.1f}`\n" - if msg.get('leverage', None) and msg.get('leverage', 1.0) != 1.0 + if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0 else "") # Check if all sell properties are available. diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index a2edcbc85..1b39a29b7 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -45,21 +45,21 @@ class Webhook(RPCHandler): try: whconfig = self._config['webhook'] if msg['type'] in [RPCMessageType.ENTRY]: - valuedict = whconfig.get('webhookentry', None) + valuedict = whconfig.get('webhookentry') elif msg['type'] in [RPCMessageType.ENTRY_CANCEL]: - valuedict = whconfig.get('webhookentrycancel', None) + valuedict = whconfig.get('webhookentrycancel') elif msg['type'] in [RPCMessageType.ENTRY_FILL]: - valuedict = whconfig.get('webhookentryfill', None) + valuedict = whconfig.get('webhookentryfill') elif msg['type'] == RPCMessageType.EXIT: - valuedict = whconfig.get('webhookexit', None) + valuedict = whconfig.get('webhookexit') elif msg['type'] == RPCMessageType.EXIT_FILL: - valuedict = whconfig.get('webhookexitfill', None) + valuedict = whconfig.get('webhookexitfill') elif msg['type'] == RPCMessageType.EXIT_CANCEL: - valuedict = whconfig.get('webhookexitcancel', None) + valuedict = whconfig.get('webhookexitcancel') elif msg['type'] in (RPCMessageType.STATUS, RPCMessageType.STARTUP, RPCMessageType.WARNING): - valuedict = whconfig.get('webhookstatus', None) + valuedict = whconfig.get('webhookstatus') else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) if not valuedict: diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 0c2197917..14e5a6743 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -131,9 +131,9 @@ class Wallets: if isinstance(balances[currency], dict): self._wallets[currency] = Wallet( currency, - balances[currency].get('free', None), - balances[currency].get('used', None), - balances[currency].get('total', None) + balances[currency].get('free'), + balances[currency].get('used'), + balances[currency].get('total') ) # Remove currencies no longer in get_balances output for currency in deepcopy(self._wallets): From 9488e8992dea615420b4712be52774b8995bec5e Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:24:52 +0100 Subject: [PATCH 002/162] First commit for integrating buy_reasons into FT --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 14 +- freqtrade/commands/cli_options.py | 31 +++ freqtrade/configuration/configuration.py | 15 ++ freqtrade/data/entryexitanalysis.py | 258 +++++++++++++++++++++++ 5 files changed, 317 insertions(+), 2 deletions(-) create mode 100755 freqtrade/data/entryexitanalysis.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 0e637c487..d93ed1e09 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -6,6 +6,7 @@ Contains all start-commands, subcommands and CLI Interface creation. Note: Be careful with file-scoped imports in these subfiles. as they are parsed on startup, nothing containing optional modules should be loaded. """ +from freqtrade.commands.analyze_commands import start_analysis_entries_exits from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 815e28175..4dd0141fa 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,6 +101,9 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] +ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", + "exit_reason_list", "indicator_list"] + NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter", @@ -182,8 +185,9 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_backtesting, start_backtesting_show, - start_convert_data, start_convert_db, start_convert_trades, + from freqtrade.commands import (start_analysis_entries_exits, start_backtesting, + start_backtesting_show, start_convert_data, + start_convert_db, start_convert_trades, start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, @@ -415,3 +419,9 @@ class Arguments: parents=[_common_parser]) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) + + # Add backtesting analysis subcommand + analysis_cmd = subparsers.add_parser('analysis', help='Analysis module.', + parents=[_common_parser, _strategy_parser]) + analysis_cmd.set_defaults(func=start_analysis_entries_exits) + self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index aac9f5713..f925bd699 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -614,4 +614,35 @@ AVAILABLE_CLI_OPTIONS = { "that do not contain any parameters."), action="store_true", ), + "analysis_groups": Arg( + "--analysis_groups", + help=("grouping output - ", + "0: simple wins/losses by enter tag, ", + "1: by enter_tag, ", + "2: by enter_tag and exit_tag, ", + "3: by pair and enter_tag, ", + "4: by pair, enter_ and exit_tag (this can get quite large)"), + nargs='?', + default="0,1,2", + ), + "enter_reason_list": Arg( + "--enter_reason_list", + help=("Comma separated list of entry signals to analyse. Default: all. ", + "e.g. 'entry_tag_a,entry_tag_b'"), + nargs='?', + default='all', + ), + "exit_reason_list": Arg( + "--exit_reason_list", + help=("Comma separated list of exit signals to analyse. Default: all. ", + "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), + nargs='?', + default='all', + ), + "indicator_list": Arg( + "--indicator_list", + help=("Comma separated list of indicators to analyse. ", + "e.g. 'close,rsi,bb_lowerband,profit_abs'"), + nargs='?', + ), } diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 96b585cd1..ea4bcace8 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -95,6 +95,8 @@ class Configuration: self._process_data_options(config) + self._process_analyze_options(config) + # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -433,6 +435,19 @@ class Configuration: self._args_to_config(config, argname='candle_types', logstring='Detected --candle-types: {}') + def _process_analyze_options(self, config: Dict[str, Any]) -> None: + self._args_to_config(config, argname='analysis_groups', + logstring='Analysis reason groups: {}') + + self._args_to_config(config, argname='enter_reason_list', + logstring='Analysis enter tag list: {}') + + self._args_to_config(config, argname='exit_reason_list', + logstring='Analysis exit tag list: {}') + + self._args_to_config(config, argname='indicator_list', + logstring='Analysis indicator list: {}') + def _process_runmode(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py new file mode 100755 index 000000000..62216d5ea --- /dev/null +++ b/freqtrade/data/entryexitanalysis.py @@ -0,0 +1,258 @@ +import joblib +import logging +import os + +from pathlib import Path +from typing import List, Optional + +import pandas as pd +from tabulate import tabulate + +from freqtrade.data.btanalysis import (load_backtest_data, get_latest_backtest_filename) +from freqtrade.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +def _load_signal_candles(backtest_dir: Path): + scpf = Path(backtest_dir, + os.path.splitext( + get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + ) + try: + scp = open(scpf, "rb") + signal_candles = joblib.load(scp) + logger.info(f"Loaded signal candles: {str(scpf)}") + except Exception as e: + logger.error("Cannot load signal candles from pickled results: ", e) + + return signal_candles + + +def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_candles): + analysed_trades_dict = {} + analysed_trades_dict[strategy_name] = {} + + try: + logger.info(f"Processing {strategy_name} : {len(pairlist)} pairs") + + for pair in pairlist: + if pair in signal_candles[strategy_name]: + analysed_trades_dict[strategy_name][pair] = _analyze_candles_and_indicators( + pair, + trades, + signal_candles[strategy_name][pair]) + except Exception: + pass + + return analysed_trades_dict + + +def _analyze_candles_and_indicators(pair, trades, signal_candles): + buyf = signal_candles + + if len(buyf) > 0: + buyf = buyf.set_index('date', drop=False) + trades_red = trades.loc[trades['pair'] == pair].copy() + + trades_inds = pd.DataFrame() + + if trades_red.shape[0] > 0 and buyf.shape[0] > 0: + for t, v in trades_red.open_date.items(): + allinds = buyf.loc[(buyf['date'] < v)] + if allinds.shape[0] > 0: + tmp_inds = allinds.iloc[[-1]] + + trades_red.loc[t, 'signal_date'] = tmp_inds['date'].values[0] + trades_red.loc[t, 'enter_reason'] = trades_red.loc[t, 'enter_tag'] + tmp_inds.index.rename('signal_date', inplace=True) + trades_inds = pd.concat([trades_inds, tmp_inds]) + + if 'signal_date' in trades_red: + trades_red['signal_date'] = pd.to_datetime(trades_red['signal_date'], utc=True) + trades_red.set_index('signal_date', inplace=True) + + try: + trades_red = pd.merge(trades_red, trades_inds, on='signal_date', how='outer') + except Exception as e: + print(e) + return trades_red + else: + return pd.DataFrame() + + +def _do_group_table_output(bigdf, glist): + if "0" in glist: + wins = bigdf.loc[bigdf['profit_abs'] >= 0] \ + .groupby(['enter_reason']) \ + .agg({'profit_abs': ['sum']}) + + wins.columns = ['profit_abs_wins'] + loss = bigdf.loc[bigdf['profit_abs'] < 0] \ + .groupby(['enter_reason']) \ + .agg({'profit_abs': ['sum']}) + loss.columns = ['profit_abs_loss'] + + new = bigdf.groupby(['enter_reason']).agg({'profit_abs': [ + 'count', + lambda x: sum(x > 0), + lambda x: sum(x <= 0)]}) + + new = pd.merge(new, wins, left_index=True, right_index=True) + new = pd.merge(new, loss, left_index=True, right_index=True) + + new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) + + new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100) + new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]) + new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]) + + new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', + 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] + + sortcols = ['total_num_buys'] + + _print_table(new, sortcols, show_index=True) + if "1" in glist: + new = bigdf.groupby(['enter_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['enter_reason', 'num_buys', 'profit_abs_sum', 'profit_abs_median', + 'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct', + 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + if "2" in glist: + new = bigdf.groupby(['enter_reason', 'exit_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', + 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', + 'mean_profit_pct', 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + if "3" in glist: + new = bigdf.groupby(['pair', 'enter_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['pair', 'enter_reason', 'num_buys', 'profit_abs_sum', + 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', + 'mean_profit_pct', 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + if "4" in glist: + new = bigdf.groupby(['pair', 'enter_reason', 'exit_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['pair', 'enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', + 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', + 'mean_profit_pct', 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + + +def _print_results(analysed_trades, stratname, group, + enter_reason_list, exit_reason_list, + indicator_list, columns=None): + + if columns is None: + columns = ['pair', 'open_date', 'close_date', 'profit_abs', 'enter_reason', 'exit_reason'] + + bigdf = pd.DataFrame() + for pair, trades in analysed_trades[stratname].items(): + bigdf = pd.concat([bigdf, trades], ignore_index=True) + + if bigdf.shape[0] > 0 and ('enter_reason' in bigdf.columns): + if group is not None: + glist = group.split(",") + _do_group_table_output(bigdf, glist) + + if enter_reason_list is not None and not enter_reason_list == "all": + enter_reason_list = enter_reason_list.split(",") + bigdf = bigdf.loc[(bigdf['enter_reason'].isin(enter_reason_list))] + + if exit_reason_list is not None and not exit_reason_list == "all": + exit_reason_list = exit_reason_list.split(",") + bigdf = bigdf.loc[(bigdf['exit_reason'].isin(exit_reason_list))] + + if indicator_list is not None: + if indicator_list == "all": + print(bigdf) + else: + available_inds = [] + for ind in indicator_list.split(","): + if ind in bigdf: + available_inds.append(ind) + ilist = ["pair", "enter_reason", "exit_reason"] + available_inds + print(tabulate(bigdf[ilist].sort_values(['exit_reason']), + headers='keys', tablefmt='psql', showindex=False)) + else: + print(tabulate(bigdf[columns].sort_values(['pair']), + headers='keys', tablefmt='psql', showindex=False)) + else: + print("\\_ No trades to show") + + +def _print_table(df, sortcols=None, show_index=False): + if (sortcols is not None): + data = df.sort_values(sortcols) + else: + data = df + + print( + tabulate( + data, + headers='keys', + tablefmt='psql', + showindex=show_index + ) + ) + + +def process_entry_exit_reasons(backtest_dir: Path, + pairlist: List[str], + strategy_name: str, + analysis_groups: Optional[str] = "0,1,2", + enter_reason_list: Optional[str] = "all", + exit_reason_list: Optional[str] = "all", + indicator_list: Optional[str] = None): + + try: + trades = load_backtest_data(backtest_dir, strategy_name) + except ValueError as e: + raise OperationalException(e) from e + if not trades.empty: + signal_candles = _load_signal_candles(backtest_dir) + analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, + trades, signal_candles) + _print_results(analysed_trades_dict, + strategy_name, + analysis_groups, + enter_reason_list, + exit_reason_list, + indicator_list) From a1a09a802b8232b0285dbdad1d9542936cf53232 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:34:31 +0100 Subject: [PATCH 003/162] Add analyze_commands --- freqtrade/commands/analyze_commands.py | 62 ++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 freqtrade/commands/analyze_commands.py diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py new file mode 100755 index 000000000..0bda9935a --- /dev/null +++ b/freqtrade/commands/analyze_commands.py @@ -0,0 +1,62 @@ +import logging +import os + +from pathlib import Path +from typing import Any, Dict + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode +from freqtrade.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for the entry/exit reason analysis module + :param args: Cli args from Arguments() + :param method: Bot running mode + :return: Configuration + """ + config = setup_utils_configuration(args, method) + + no_unlimited_runmodes = { + RunMode.BACKTEST: 'backtesting', + } + if method in no_unlimited_runmodes.keys(): + from freqtrade.data.btanalysis import get_latest_backtest_filename + + btp = Path(config.get('user_data_dir'), "backtest_results") + btfile = get_latest_backtest_filename(btp) + signals_file = f"{os.path.basename(os.path.splitext(btfile)[0])}_signals.pkl" + + if (not os.path.exists(Path(btp, signals_file))): + raise OperationalException( + "Cannot find latest backtest signals file. Run backtesting with --export signals." + ) + + return config + + +def start_analysis_entries_exits(args: Dict[str, Any]) -> None: + """ + Start analysis script + :param args: Cli args from Arguments() + :return: None + """ + from freqtrade.data.entryexitanalysis import process_entry_exit_reasons + + # Initialize configuration + config = setup_analyze_configuration(args, RunMode.BACKTEST) + + logger.info('Starting freqtrade in analysis mode') + + process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), + config['exchange']['pair_whitelist'], + config['strategy'], + config['analysis_groups'], + config['enter_reason_list'], + config['exit_reason_list'], + config['indicator_list'] + ) From ae1ede58da1193553d5fcb5c85c3911b6e1c8664 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:41:28 +0100 Subject: [PATCH 004/162] Fix import order --- freqtrade/commands/analyze_commands.py | 1 - freqtrade/data/entryexitanalysis.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 0bda9935a..1590dc519 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -1,6 +1,5 @@ import logging import os - from pathlib import Path from typing import Any, Dict diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 62216d5ea..e22a2475e 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -1,14 +1,13 @@ -import joblib import logging import os - from pathlib import Path from typing import List, Optional +import joblib import pandas as pd from tabulate import tabulate -from freqtrade.data.btanalysis import (load_backtest_data, get_latest_backtest_filename) +from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data from freqtrade.exceptions import OperationalException From 80c6190c055f606510abef151886f0607f3fd88a Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:55:59 +0100 Subject: [PATCH 005/162] Fix analyze_commands setup --- freqtrade/commands/analyze_commands.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 1590dc519..a4b3d3f52 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -26,11 +26,10 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s if method in no_unlimited_runmodes.keys(): from freqtrade.data.btanalysis import get_latest_backtest_filename - btp = Path(config.get('user_data_dir'), "backtest_results") - btfile = get_latest_backtest_filename(btp) + btfile = get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results') signals_file = f"{os.path.basename(os.path.splitext(btfile)[0])}_signals.pkl" - if (not os.path.exists(Path(btp, signals_file))): + if (not os.path.exists(config['user_data_dir'] / 'backtest_results' / signals_file)): raise OperationalException( "Cannot find latest backtest signals file. Run backtesting with --export signals." ) From 8c03ebb78ff637a4545391c2ac62510ba1a1c4f1 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 12:48:13 +0100 Subject: [PATCH 006/162] Fix group 0 table, add pathlib.Path use --- freqtrade/commands/analyze_commands.py | 14 ++++++++++---- freqtrade/commands/cli_options.py | 1 + freqtrade/data/entryexitanalysis.py | 12 +++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index a4b3d3f52..73ae19eaf 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -1,5 +1,4 @@ import logging -import os from pathlib import Path from typing import Any, Dict @@ -26,14 +25,19 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s if method in no_unlimited_runmodes.keys(): from freqtrade.data.btanalysis import get_latest_backtest_filename - btfile = get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results') - signals_file = f"{os.path.basename(os.path.splitext(btfile)[0])}_signals.pkl" + btfile = Path(get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results')) + signals_file = f"{btfile.stem}_signals.pkl" - if (not os.path.exists(config['user_data_dir'] / 'backtest_results' / signals_file)): + if (not (config['user_data_dir'] / 'backtest_results' / signals_file).exists()): raise OperationalException( "Cannot find latest backtest signals file. Run backtesting with --export signals." ) + if ('strategy' not in config): + raise OperationalException( + "No strategy defined. Use --strategy or supply in config." + ) + return config @@ -48,6 +52,8 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: # Initialize configuration config = setup_analyze_configuration(args, RunMode.BACKTEST) + print(config) + logger.info('Starting freqtrade in analysis mode') process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f925bd699..f76f3688c 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -644,5 +644,6 @@ AVAILABLE_CLI_OPTIONS = { help=("Comma separated list of indicators to analyse. ", "e.g. 'close,rsi,bb_lowerband,profit_abs'"), nargs='?', + default='', ), } diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index e22a2475e..8bfc940dc 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -97,15 +97,12 @@ def _do_group_table_output(bigdf, glist): 'count', lambda x: sum(x > 0), lambda x: sum(x <= 0)]}) - - new = pd.merge(new, wins, left_index=True, right_index=True) - new = pd.merge(new, loss, left_index=True, right_index=True) + new = pd.concat([new, wins, loss], axis=1).fillna(0) new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) - - new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100) - new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]) - new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]) + new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100).fillna(0) + new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0) + new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0) new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] @@ -249,6 +246,7 @@ def process_entry_exit_reasons(backtest_dir: Path, signal_candles = _load_signal_candles(backtest_dir) analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, trades, signal_candles) + _print_results(analysed_trades_dict, strategy_name, analysis_groups, From 3adda84b96bdcde1909a6cecb12ef9b3fbd9296c Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 20:27:15 +0100 Subject: [PATCH 007/162] Update docs, add test --- docs/advanced-backtesting.md | 16 ++-- freqtrade/commands/analyze_commands.py | 2 - freqtrade/data/entryexitanalysis.py | 17 ++-- tests/data/test_entryexitanalysis.py | 94 +++++++++++++++++++++++ tests/strategy/strats/strategy_test_v3.py | 9 ++- 5 files changed, 117 insertions(+), 21 deletions(-) create mode 100755 tests/data/test_entryexitanalysis.py diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 2a484da69..4b40bad8e 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -22,23 +22,19 @@ DataFrame of the candles that resulted in buy signals. Depending on how many buy makes, this file may get quite large, so periodically check your `user_data/backtest_results` folder to delete old exports. -To analyze the buy tags, we need to use the `buy_reasons.py` script from -[froggleston's repo](https://github.com/froggleston/freqtrade-buyreasons). Follow the instructions -in their README to copy the script into your `freqtrade/scripts/` folder. - Before running your next backtest, make sure you either delete your old backtest results or run backtesting with the `--cache none` option to make sure no cached results are used. If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the `user_data/backtest_results` folder. -Now run the `buy_reasons.py` script, supplying a few options: +To analyze the entry/exit tags, we now need to use the `freqtrade analysis` command: ``` bash -python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 +freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 ``` -The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) +The `--analysis_groups` option is used to specify the various tabular outputs, ranging from the simplest (0) to the most detailed per pair, per buy and per sell tag (4). More options are available by running with the `-h` option. @@ -54,18 +50,18 @@ To show only certain buy and sell tags in the displayed output, use the followin For example: ```bash -python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" +freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" ``` ### Outputting signal candle indicators -The real power of the buy_reasons.py script comes from the ability to print out the indicator +The real power of `freqtrade analysis` comes from the ability to print out the indicator values present on signal candles to allow fine-grained investigation and tuning of buy signal indicators. To print out a column for a given set of indicators, use the `--indicator-list` option: ```bash -python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" ``` The indicators have to be present in your strategy's main DataFrame (either for your main diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 73ae19eaf..56330bed3 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -52,8 +52,6 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: # Initialize configuration config = setup_analyze_configuration(args, RunMode.BACKTEST) - print(config) - logger.info('Starting freqtrade in analysis mode') process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 8bfc940dc..9d6c470da 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -15,10 +15,18 @@ logger = logging.getLogger(__name__) def _load_signal_candles(backtest_dir: Path): - scpf = Path(backtest_dir, - os.path.splitext( - get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" - ) + + if backtest_dir.is_dir(): + scpf = Path(backtest_dir, + os.path.splitext( + get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + ) + else: + scpf = Path(os.path.splitext( + get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + ) + + print(scpf) try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) @@ -246,7 +254,6 @@ def process_entry_exit_reasons(backtest_dir: Path, signal_candles = _load_signal_candles(backtest_dir) analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, trades, signal_candles) - _print_results(analysed_trades_dict, strategy_name, analysis_groups, diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py new file mode 100755 index 000000000..548cd88b9 --- /dev/null +++ b/tests/data/test_entryexitanalysis.py @@ -0,0 +1,94 @@ +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +import pandas as pd + +from freqtrade.commands.analyze_commands import start_analysis_entries_exits +from freqtrade.commands.optimize_commands import start_backtesting +from freqtrade.enums import ExitType +from tests.conftest import get_args, patch_exchange, patched_configuration_load_config_file + + +def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, capsys): + default_conf.update({ + "use_exit_signal": True, + "exit_profit_only": False, + "exit_profit_offset": 0.0, + "ignore_roi_if_entry_signal": False, + 'analysis_groups': "0", + 'enter_reason_list': "all", + 'exit_reason_list': "all", + 'indicator_list': "bb_upperband,ema_10" + }) + patch_exchange(mocker) + result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], + 'profit_ratio': [0.0, 0.0], + 'profit_abs': [0.0, 0.0], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', ], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', ], utc=True), + 'trade_duration': [235, 40], + 'is_open': [False, False], + 'stake_amount': [0.01, 0.01], + 'open_rate': [0.104445, 0.10302485], + 'close_rate': [0.104969, 0.103541], + "is_short": [False, False], + 'enter_tag': ["enter_tag_long", "enter_tag_long"], + 'exit_reason': [ExitType.ROI, ExitType.ROI] + }) + + backtestmock = MagicMock(side_effect=[ + { + 'results': result1, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'timedout_entry_orders': 0, + 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, + 'final_balance': 1000, + } + ]) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['ETH/BTC', 'LTC/BTC', 'DASH/BTC'])) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, default_conf) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '5m', + '--timerange', '1515560100-1517287800', + '--export', 'signals', + '--cache', 'none', + '--strategy-list', + 'StrategyTestV3', + ] + args = get_args(args) + start_backtesting(args) + + captured = capsys.readouterr() + assert 'BACKTESTING REPORT' in captured.out + assert 'EXIT REASON STATS' in captured.out + assert 'LEFT OPEN TRADES REPORT' in captured.out + + args = [ + 'analysis', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--analysis_groups', '0', + '--strategy', + 'StrategyTestV3', + ] + args = get_args(args) + start_analysis_entries_exits(args) + + captured = capsys.readouterr() + assert 'enter_tag_long' in captured.out diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index df83d3663..f1c9d8e99 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -143,12 +143,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 65) & (dataframe['plus_di'] > self.buy_plusdi.value) ), - 'enter_long'] = 1 + ['enter_long', 'enter_tag']] = 1, 'enter_tag_long' + dataframe.loc[ ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) ), - 'enter_short'] = 1 + ['enter_short', 'enter_tag']] = 1, 'enter_tag_short' return dataframe @@ -166,13 +167,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 70) & (dataframe['minus_di'] > self.sell_minusdi.value) ), - 'exit_long'] = 1 + ['exit_long', 'exit_tag']] = 1, 'exit_tag_long' dataframe.loc[ ( qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) ), - 'exit_short'] = 1 + ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' return dataframe From 22b9805e472f44ce1da2928b6d8177c293012048 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 21:04:23 +0100 Subject: [PATCH 008/162] Fix all tests --- tests/data/test_entryexitanalysis.py | 4 +- tests/rpc/test_rpc_apiserver.py | 6 +- tests/strategy/strats/strategy_test_v3.py | 8 +- .../strats/strategy_test_v3_analysis.py | 195 ++++++++++++++++++ tests/strategy/test_strategy_loading.py | 6 +- 5 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 tests/strategy/strats/strategy_test_v3_analysis.py diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 548cd88b9..151fc3ff8 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -69,7 +69,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap '--export', 'signals', '--cache', 'none', '--strategy-list', - 'StrategyTestV3', + 'StrategyTestV3Analysis', ] args = get_args(args) start_backtesting(args) @@ -85,7 +85,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap '--datadir', str(testdatadir), '--analysis_groups', '0', '--strategy', - 'StrategyTestV3', + 'StrategyTestV3Analysis', ] args = get_args(args) start_analysis_entries_exits(args) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 03ba895a1..c887e7776 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1384,12 +1384,16 @@ def test_api_strategies(botclient): rc = client_get(client, f"{BASE_URI}/strategies") assert_response(rc) + + print(rc.json()) + assert rc.json() == {'strategies': [ 'HyperoptableStrategy', 'InformativeDecoratorTest', 'StrategyTestV2', 'StrategyTestV3', - 'StrategyTestV3Futures', + 'StrategyTestV3Analysis', + 'StrategyTestV3Futures' ]} diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index f1c9d8e99..9ca2471bd 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -143,13 +143,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 65) & (dataframe['plus_di'] > self.buy_plusdi.value) ), - ['enter_long', 'enter_tag']] = 1, 'enter_tag_long' + 'enter_long'] = 1 dataframe.loc[ ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) ), - ['enter_short', 'enter_tag']] = 1, 'enter_tag_short' + 'enter_short'] = 1 return dataframe @@ -167,13 +167,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 70) & (dataframe['minus_di'] > self.sell_minusdi.value) ), - ['exit_long', 'exit_tag']] = 1, 'exit_tag_long' + 'exit_long'] = 1 dataframe.loc[ ( qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) ), - ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' + 'exit_short'] = 1 return dataframe diff --git a/tests/strategy/strats/strategy_test_v3_analysis.py b/tests/strategy/strats/strategy_test_v3_analysis.py new file mode 100644 index 000000000..b237f548f --- /dev/null +++ b/tests/strategy/strats/strategy_test_v3_analysis.py @@ -0,0 +1,195 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +from datetime import datetime + +import talib.abstract as ta +from pandas import DataFrame + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.persistence import Trade +from freqtrade.strategy import (BooleanParameter, DecimalParameter, IntParameter, IStrategy, + RealParameter) + + +class StrategyTestV3Analysis(IStrategy): + """ + Strategy used by tests freqtrade bot. + Please do not modify this strategy, it's intended for internal use only. + Please look at the SampleStrategy in the user_data/strategy directory + or strategy repository https://github.com/freqtrade/freqtrade-strategies + for samples and inspiration. + """ + INTERFACE_VERSION = 3 + + # Minimal ROI designed for the strategy + minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + stoploss = -0.10 + + # Optimal timeframe for the strategy + timeframe = '5m' + + # Optional order type mapping + order_types = { + 'entry': 'limit', + 'exit': 'limit', + 'stoploss': 'limit', + 'stoploss_on_exchange': False + } + + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 20 + + # Optional time in force for orders + order_time_in_force = { + 'entry': 'gtc', + 'exit': 'gtc', + } + + buy_params = { + 'buy_rsi': 35, + # Intentionally not specified, so "default" is tested + # 'buy_plusdi': 0.4 + } + + sell_params = { + 'sell_rsi': 74, + 'sell_minusdi': 0.4 + } + + buy_rsi = IntParameter([0, 50], default=30, space='buy') + buy_plusdi = RealParameter(low=0, high=1, default=0.5, space='buy') + sell_rsi = IntParameter(low=50, high=100, default=70, space='sell') + sell_minusdi = DecimalParameter(low=0, high=1, default=0.5001, decimals=3, space='sell', + load=False) + protection_enabled = BooleanParameter(default=True) + protection_cooldown_lookback = IntParameter([0, 50], default=30) + + # TODO: Can this work with protection tests? (replace HyperoptableStrategy implicitly ... ) + # @property + # def protections(self): + # prot = [] + # if self.protection_enabled.value: + # prot.append({ + # "method": "CooldownPeriod", + # "stop_duration_candles": self.protection_cooldown_lookback.value + # }) + # return prot + + bot_started = False + + def bot_start(self): + self.bot_started = True + + def informative_pairs(self): + + return [] + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # Minus Directional Indicator / Movement + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + # EMA - Exponential Moving Average + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + + return dataframe + + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + dataframe.loc[ + ( + (dataframe['rsi'] < self.buy_rsi.value) & + (dataframe['fastd'] < 35) & + (dataframe['adx'] > 30) & + (dataframe['plus_di'] > self.buy_plusdi.value) + ) | + ( + (dataframe['adx'] > 65) & + (dataframe['plus_di'] > self.buy_plusdi.value) + ), + ['enter_long', 'enter_tag']] = 1, 'enter_tag_long' + + dataframe.loc[ + ( + qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) + ), + ['enter_short', 'enter_tag']] = 1, 'enter_tag_short' + + return dataframe + + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe.loc[ + ( + ( + (qtpylib.crossed_above(dataframe['rsi'], self.sell_rsi.value)) | + (qtpylib.crossed_above(dataframe['fastd'], 70)) + ) & + (dataframe['adx'] > 10) & + (dataframe['minus_di'] > 0) + ) | + ( + (dataframe['adx'] > 70) & + (dataframe['minus_di'] > self.sell_minusdi.value) + ), + ['exit_long', 'exit_tag']] = 1, 'exit_tag_long' + + dataframe.loc[ + ( + qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) + ), + ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' + + return dataframe + + def leverage(self, pair: str, current_time: datetime, current_rate: float, + proposed_leverage: float, max_leverage: float, side: str, + **kwargs) -> float: + # Return 3.0 in all cases. + # Bot-logic must make sure it's an allowed leverage and eventually adjust accordingly. + + return 3.0 + + def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, min_stake: float, max_stake: float, **kwargs): + + if current_profit < -0.0075: + orders = trade.select_filled_orders(trade.entry_side) + return round(orders[0].cost, 0) + + return None diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 919a4bd00..666ae2b05 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -34,7 +34,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 5 + assert len(strategies) == 6 assert isinstance(strategies[0], dict) @@ -42,10 +42,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 6 + assert len(strategies) == 7 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 5 + assert len([x for x in strategies if x['class'] is not None]) == 6 assert len([x for x in strategies if x['class'] is None]) == 1 From edd474e663b950ade4b4fe172846a8045b013b3e Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 21:21:20 +0100 Subject: [PATCH 009/162] Another test fix attempt --- .../strats/strategy_test_v3_analysis.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/tests/strategy/strats/strategy_test_v3_analysis.py b/tests/strategy/strats/strategy_test_v3_analysis.py index b237f548f..290fef156 100644 --- a/tests/strategy/strats/strategy_test_v3_analysis.py +++ b/tests/strategy/strats/strategy_test_v3_analysis.py @@ -1,12 +1,9 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -from datetime import datetime - import talib.abstract as ta from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.persistence import Trade from freqtrade.strategy import (BooleanParameter, DecimalParameter, IntParameter, IStrategy, RealParameter) @@ -176,20 +173,3 @@ class StrategyTestV3Analysis(IStrategy): ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' return dataframe - - def leverage(self, pair: str, current_time: datetime, current_rate: float, - proposed_leverage: float, max_leverage: float, side: str, - **kwargs) -> float: - # Return 3.0 in all cases. - # Bot-logic must make sure it's an allowed leverage and eventually adjust accordingly. - - return 3.0 - - def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, min_stake: float, max_stake: float, **kwargs): - - if current_profit < -0.0075: - orders = trade.select_filled_orders(trade.entry_side) - return round(orders[0].cost, 0) - - return None From 2873ca6d38329245f6aedb84f93f94f1a992eb77 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 25 May 2022 09:57:12 +0100 Subject: [PATCH 010/162] Add cleanup, adjust _print_table for indicators, add rsi to test output --- freqtrade/data/entryexitanalysis.py | 7 ++----- tests/data/test_entryexitanalysis.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 9d6c470da..192d666ae 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -26,7 +26,6 @@ def _load_signal_candles(backtest_dir: Path): get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" ) - print(scpf) try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) @@ -213,11 +212,9 @@ def _print_results(analysed_trades, stratname, group, if ind in bigdf: available_inds.append(ind) ilist = ["pair", "enter_reason", "exit_reason"] + available_inds - print(tabulate(bigdf[ilist].sort_values(['exit_reason']), - headers='keys', tablefmt='psql', showindex=False)) + _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) else: - print(tabulate(bigdf[columns].sort_values(['pair']), - headers='keys', tablefmt='psql', showindex=False)) + _print_table(bigdf[columns], sortcols=['pair'], show_index=False) else: print("\\_ No trades to show") diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 151fc3ff8..70ba5fa21 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -2,13 +2,22 @@ from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pandas as pd +import pytest from freqtrade.commands.analyze_commands import start_analysis_entries_exits from freqtrade.commands.optimize_commands import start_backtesting from freqtrade.enums import ExitType +from freqtrade.optimize.backtesting import Backtesting from tests.conftest import get_args, patch_exchange, patched_configuration_load_config_file +@pytest.fixture(autouse=True) +def backtesting_cleanup() -> None: + yield None + + Backtesting.cleanup() + + def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, capsys): default_conf.update({ "use_exit_signal": True, @@ -18,7 +27,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap 'analysis_groups': "0", 'enter_reason_list': "all", 'exit_reason_list': "all", - 'indicator_list': "bb_upperband,ema_10" + 'indicator_list': "rsi" }) patch_exchange(mocker) result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], @@ -84,6 +93,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap '--config', 'config.json', '--datadir', str(testdatadir), '--analysis_groups', '0', + '--indicator_list', 'rsi', '--strategy', 'StrategyTestV3Analysis', ] @@ -92,3 +102,6 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap captured = capsys.readouterr() assert 'enter_tag_long' in captured.out + assert '34.049' in captured.out + + Backtesting.cleanup() From f5c2930889c7d9de7e999817389bff185adb117c Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 25 May 2022 09:58:38 +0100 Subject: [PATCH 011/162] Presume that pytest will call the cleanup call --- tests/data/test_entryexitanalysis.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 70ba5fa21..3ee986600 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -12,7 +12,7 @@ from tests.conftest import get_args, patch_exchange, patched_configuration_load_ @pytest.fixture(autouse=True) -def backtesting_cleanup() -> None: +def entryexitanalysis_cleanup() -> None: yield None Backtesting.cleanup() @@ -103,5 +103,3 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap captured = capsys.readouterr() assert 'enter_tag_long' in captured.out assert '34.049' in captured.out - - Backtesting.cleanup() From 21e6c14e1e80e65eff1d50e7d85e0aa330b7983c Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 25 May 2022 10:08:03 +0100 Subject: [PATCH 012/162] Final test changes --- tests/data/test_entryexitanalysis.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 3ee986600..9ae89a2f8 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -102,4 +102,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap captured = capsys.readouterr() assert 'enter_tag_long' in captured.out + assert 'ETH/BTC' in captured.out assert '34.049' in captured.out + assert 'LTC/BTC' in captured.out + assert '54.3204' in captured.out From 145faf98178e5b8bc69c7cd1dba3a01eda9d2d7e Mon Sep 17 00:00:00 2001 From: froggleston Date: Thu, 26 May 2022 11:06:38 +0100 Subject: [PATCH 013/162] Use tmpdir for testing --- tests/data/test_entryexitanalysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 9ae89a2f8..ed0bab76b 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -18,7 +18,7 @@ def entryexitanalysis_cleanup() -> None: Backtesting.cleanup() -def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, capsys): +def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmpdir, capsys): default_conf.update({ "use_exit_signal": True, "exit_profit_only": False, @@ -72,6 +72,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap 'backtesting', '--config', 'config.json', '--datadir', str(testdatadir), + '--user-data-dir', str(tmpdir), '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), '--timeframe', '5m', '--timerange', '1515560100-1517287800', @@ -92,6 +93,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap 'analysis', '--config', 'config.json', '--datadir', str(testdatadir), + '--user-data-dir', str(tmpdir), '--analysis_groups', '0', '--indicator_list', 'rsi', '--strategy', From 43b7955fc2b43a78102f802ed5dee9e348564823 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 May 2022 19:37:55 +0200 Subject: [PATCH 014/162] Fully rely on pathlib --- freqtrade/data/entryexitanalysis.py | 10 +++------- tests/rpc/test_rpc_apiserver.py | 2 -- tests/strategy/strats/strategy_test_v3.py | 1 - 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 192d666ae..3c83d4abf 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -1,5 +1,4 @@ import logging -import os from pathlib import Path from typing import List, Optional @@ -18,14 +17,11 @@ def _load_signal_candles(backtest_dir: Path): if backtest_dir.is_dir(): scpf = Path(backtest_dir, - os.path.splitext( - get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl" ) else: - scpf = Path(os.path.splitext( - get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" - ) - + scpf = Path(Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl") + print(scpf) try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index c887e7776..8b3ac18ac 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1385,8 +1385,6 @@ def test_api_strategies(botclient): assert_response(rc) - print(rc.json()) - assert rc.json() == {'strategies': [ 'HyperoptableStrategy', 'InformativeDecoratorTest', diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 9ca2471bd..df83d3663 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -144,7 +144,6 @@ class StrategyTestV3(IStrategy): (dataframe['plus_di'] > self.buy_plusdi.value) ), 'enter_long'] = 1 - dataframe.loc[ ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) From e7c5818d1645af2e373f346cee98e3a69e47395b Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 11:20:11 +0100 Subject: [PATCH 015/162] First pass changes for cleaning up --- freqtrade/commands/arguments.py | 6 +- freqtrade/commands/cli_options.py | 8 +- freqtrade/data/entryexitanalysis.py | 145 ++++++++++++--------------- tests/data/test_entryexitanalysis.py | 12 +-- 4 files changed, 75 insertions(+), 96 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 4dd0141fa..679193e49 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,8 +101,8 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] -ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", - "exit_reason_list", "indicator_list"] +ARGS_ANALYZE_ENTRIES_EXITS = ["analysis-groups", "enter-reason-list", + "exit-reason-list", "indicator-list"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", @@ -421,7 +421,7 @@ class Arguments: self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('analysis', help='Analysis module.', + analysis_cmd = subparsers.add_parser('analysis', help='Backtest Analysis module.', parents=[_common_parser, _strategy_parser]) analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f76f3688c..ce7320b95 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -615,7 +615,7 @@ AVAILABLE_CLI_OPTIONS = { action="store_true", ), "analysis_groups": Arg( - "--analysis_groups", + "--analysis-groups", help=("grouping output - ", "0: simple wins/losses by enter tag, ", "1: by enter_tag, ", @@ -626,21 +626,21 @@ AVAILABLE_CLI_OPTIONS = { default="0,1,2", ), "enter_reason_list": Arg( - "--enter_reason_list", + "--enter-reason-list", help=("Comma separated list of entry signals to analyse. Default: all. ", "e.g. 'entry_tag_a,entry_tag_b'"), nargs='?', default='all', ), "exit_reason_list": Arg( - "--exit_reason_list", + "--exit-reason-list", help=("Comma separated list of exit signals to analyse. Default: all. ", "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), nargs='?', default='all', ), "indicator_list": Arg( - "--indicator_list", + "--indicator-list", help=("Comma separated list of indicators to analyse. ", "e.g. 'close,rsi,bb_lowerband,profit_abs'"), nargs='?', diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 192d666ae..53a256633 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -7,7 +7,8 @@ import joblib import pandas as pd from tabulate import tabulate -from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data +from freqtrade.data.btanalysis import (get_latest_backtest_filename, load_backtest_data, + load_backtest_stats) from freqtrade.exceptions import OperationalException @@ -49,8 +50,8 @@ def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_cand pair, trades, signal_candles[strategy_name][pair]) - except Exception: - pass + except Exception as e: + print(f"Cannot process entry/exit reasons for {strategy_name}: ", e) return analysed_trades_dict @@ -82,104 +83,79 @@ def _analyze_candles_and_indicators(pair, trades, signal_candles): try: trades_red = pd.merge(trades_red, trades_inds, on='signal_date', how='outer') except Exception as e: - print(e) + raise e return trades_red else: return pd.DataFrame() def _do_group_table_output(bigdf, glist): - if "0" in glist: - wins = bigdf.loc[bigdf['profit_abs'] >= 0] \ - .groupby(['enter_reason']) \ - .agg({'profit_abs': ['sum']}) + for g in glist: + # 0: summary wins/losses grouped by enter tag + if g == "0": + group_mask = ['enter_reason'] + wins = bigdf.loc[bigdf['profit_abs'] >= 0] \ + .groupby(group_mask) \ + .agg({'profit_abs': ['sum']}) - wins.columns = ['profit_abs_wins'] - loss = bigdf.loc[bigdf['profit_abs'] < 0] \ - .groupby(['enter_reason']) \ - .agg({'profit_abs': ['sum']}) - loss.columns = ['profit_abs_loss'] + wins.columns = ['profit_abs_wins'] + loss = bigdf.loc[bigdf['profit_abs'] < 0] \ + .groupby(group_mask) \ + .agg({'profit_abs': ['sum']}) + loss.columns = ['profit_abs_loss'] - new = bigdf.groupby(['enter_reason']).agg({'profit_abs': [ - 'count', - lambda x: sum(x > 0), - lambda x: sum(x <= 0)]}) - new = pd.concat([new, wins, loss], axis=1).fillna(0) + new = bigdf.groupby(group_mask).agg({'profit_abs': [ + 'count', + lambda x: sum(x > 0), + lambda x: sum(x <= 0)]}) + new = pd.concat([new, wins, loss], axis=1).fillna(0) - new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) - new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100).fillna(0) - new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0) - new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0) + new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) + new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100).fillna(0) + new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0) + new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0) - new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', - 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] + new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', + 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] - sortcols = ['total_num_buys'] + sortcols = ['total_num_buys'] - _print_table(new, sortcols, show_index=True) - if "1" in glist: - new = bigdf.groupby(['enter_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], - 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['enter_reason', 'num_buys', 'profit_abs_sum', 'profit_abs_median', - 'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct', - 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] + _print_table(new, sortcols, show_index=True) - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 - - _print_table(new, sortcols) - if "2" in glist: - new = bigdf.groupby(['enter_reason', 'exit_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], - 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', - 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', - 'mean_profit_pct', 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] - - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 - - _print_table(new, sortcols) - if "3" in glist: - new = bigdf.groupby(['pair', 'enter_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + else: + agg_mask = {'profit_abs': ['count', 'sum', 'median', 'mean'], 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['pair', 'enter_reason', 'num_buys', 'profit_abs_sum', - 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', - 'mean_profit_pct', 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] + agg_cols = ['num_buys', 'profit_abs_sum', 'profit_abs_median', + 'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct', + 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 + # 1: profit summaries grouped by enter_tag + if g == "1": + group_mask = ['enter_reason'] - _print_table(new, sortcols) - if "4" in glist: - new = bigdf.groupby(['pair', 'enter_reason', 'exit_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], - 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['pair', 'enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', - 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', - 'mean_profit_pct', 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] + # 2: profit summaries grouped by enter_tag and exit_tag + if g == "2": + group_mask = ['enter_reason', 'exit_reason'] - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 + # 3: profit summaries grouped by pair and enter_tag + if g == "3": + group_mask = ['pair', 'enter_reason'] - _print_table(new, sortcols) + # 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large) + if g == "4": + group_mask = ['pair', 'enter_reason', 'exit_reason'] + + new = bigdf.groupby(group_mask).agg(agg_mask).reset_index() + new.columns = group_mask + agg_cols + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) -def _print_results(analysed_trades, stratname, group, +def _print_results(analysed_trades, stratname, analysis_groups, enter_reason_list, exit_reason_list, indicator_list, columns=None): @@ -191,8 +167,8 @@ def _print_results(analysed_trades, stratname, group, bigdf = pd.concat([bigdf, trades], ignore_index=True) if bigdf.shape[0] > 0 and ('enter_reason' in bigdf.columns): - if group is not None: - glist = group.split(",") + if analysis_groups is not None: + glist = analysis_groups.split(",") _do_group_table_output(bigdf, glist) if enter_reason_list is not None and not enter_reason_list == "all": @@ -244,6 +220,9 @@ def process_entry_exit_reasons(backtest_dir: Path, indicator_list: Optional[str] = None): try: + bt_stats = load_backtest_stats(backtest_dir) + logger.info(bt_stats) + # strategy_name = bt_stats['something'] trades = load_backtest_data(backtest_dir, strategy_name) except ValueError as e: raise OperationalException(e) from e diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index ed0bab76b..90da80ce9 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -24,10 +24,10 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp "exit_profit_only": False, "exit_profit_offset": 0.0, "ignore_roi_if_entry_signal": False, - 'analysis_groups': "0", - 'enter_reason_list': "all", - 'exit_reason_list': "all", - 'indicator_list': "rsi" + 'analysis-groups': "0", + 'enter-reason-list': "all", + 'exit-reason-list': "all", + 'indicator-list': "rsi" }) patch_exchange(mocker) result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], @@ -94,8 +94,8 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), - '--analysis_groups', '0', - '--indicator_list', 'rsi', + '--analysis-groups', '0', + '--indicator-list', 'rsi', '--strategy', 'StrategyTestV3Analysis', ] From df1c36e5aa7085a78ee06ab2cdd1558b0ccd7331 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 11:54:27 +0100 Subject: [PATCH 016/162] Change command name, use load_backtest_stats for strategy resolving --- freqtrade/commands/analyze_commands.py | 1 - freqtrade/commands/arguments.py | 7 +++--- freqtrade/data/entryexitanalysis.py | 35 ++++++++++++-------------- tests/data/test_entryexitanalysis.py | 13 ++++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 56330bed3..2fa13f683 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -56,7 +56,6 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), config['exchange']['pair_whitelist'], - config['strategy'], config['analysis_groups'], config['enter_reason_list'], config['exit_reason_list'], diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 679193e49..d5831a2ac 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,8 +101,8 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] -ARGS_ANALYZE_ENTRIES_EXITS = ["analysis-groups", "enter-reason-list", - "exit-reason-list", "indicator-list"] +ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", + "exit_reason_list", "indicator_list"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", @@ -421,7 +421,8 @@ class Arguments: self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('analysis', help='Backtest Analysis module.', + analysis_cmd = subparsers.add_parser('backtesting-analysis', + help='Backtest Analysis module.', parents=[_common_parser, _strategy_parser]) analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 1ee6eea42..15ac6ba09 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -15,14 +15,13 @@ logger = logging.getLogger(__name__) def _load_signal_candles(backtest_dir: Path): - if backtest_dir.is_dir(): scpf = Path(backtest_dir, Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl" ) else: scpf = Path(Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl") - print(scpf) + try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) @@ -154,7 +153,6 @@ def _do_group_table_output(bigdf, glist): def _print_results(analysed_trades, stratname, analysis_groups, enter_reason_list, exit_reason_list, indicator_list, columns=None): - if columns is None: columns = ['pair', 'open_date', 'close_date', 'profit_abs', 'enter_reason', 'exit_reason'] @@ -209,26 +207,25 @@ def _print_table(df, sortcols=None, show_index=False): def process_entry_exit_reasons(backtest_dir: Path, pairlist: List[str], - strategy_name: str, analysis_groups: Optional[str] = "0,1,2", enter_reason_list: Optional[str] = "all", exit_reason_list: Optional[str] = "all", indicator_list: Optional[str] = None): - try: - bt_stats = load_backtest_stats(backtest_dir) - logger.info(bt_stats) - # strategy_name = bt_stats['something'] - trades = load_backtest_data(backtest_dir, strategy_name) + backtest_stats = load_backtest_stats(backtest_dir) + for strategy_name, results in backtest_stats['strategy'].items(): + trades = load_backtest_data(backtest_dir, strategy_name) + + if not trades.empty: + signal_candles = _load_signal_candles(backtest_dir) + analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, + trades, signal_candles) + _print_results(analysed_trades_dict, + strategy_name, + analysis_groups, + enter_reason_list, + exit_reason_list, + indicator_list) + except ValueError as e: raise OperationalException(e) from e - if not trades.empty: - signal_candles = _load_signal_candles(backtest_dir) - analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, - trades, signal_candles) - _print_results(analysed_trades_dict, - strategy_name, - analysis_groups, - enter_reason_list, - exit_reason_list, - indicator_list) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 90da80ce9..eadf79179 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -24,10 +24,6 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp "exit_profit_only": False, "exit_profit_offset": 0.0, "ignore_roi_if_entry_signal": False, - 'analysis-groups': "0", - 'enter-reason-list': "all", - 'exit-reason-list': "all", - 'indicator-list': "rsi" }) patch_exchange(mocker) result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], @@ -89,8 +85,15 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert 'EXIT REASON STATS' in captured.out assert 'LEFT OPEN TRADES REPORT' in captured.out + default_conf.update({ + 'analysis_groups': "0", + 'enter_reason_list': "all", + 'exit_reason_list': "all", + 'indicator_list': "rsi" + }) + args = [ - 'analysis', + 'backtesting-analysis', '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), From 24b02127ec03e7c7e05b13fb450310980a7663a4 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 15:42:34 +0100 Subject: [PATCH 017/162] Update docs --- docs/advanced-backtesting.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 4b40bad8e..7f2be1f1a 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -28,40 +28,47 @@ backtesting with the `--cache none` option to make sure no cached results are us If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the `user_data/backtest_results` folder. -To analyze the entry/exit tags, we now need to use the `freqtrade analysis` command: +To analyze the entry/exit tags, we now need to use the `freqtrade backtesting-analysis` command: ``` bash -freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 +freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 ``` -The `--analysis_groups` option is used to specify the various tabular outputs, ranging from the simplest (0) -to the most detailed per pair, per buy and per sell tag (4). More options are available by -running with the `-h` option. +This command will read from the last backtesting results. The `--analysis-groups` option is +used to specify the various tabular outputs showing the profit fo each group or trade, +ranging from the simplest (0) to the most detailed per pair, per buy and per sell tag (4): + +* 1: profit summaries grouped by enter_tag +* 2: profit summaries grouped by enter_tag and exit_tag +* 3: profit summaries grouped by pair and enter_tag +* 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large) + +More options are available by running with the `-h` option. ### Tuning the buy tags and sell tags to display To show only certain buy and sell tags in the displayed output, use the following two options: ``` ---enter_reason_list : Comma separated list of enter signals to analyse. Default: "all" ---exit_reason_list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss" +--enter-reason-list : Comma separated list of enter signals to analyse. Default: "all" +--exit-reason-list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss" ``` For example: ```bash -freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" +freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" ``` ### Outputting signal candle indicators -The real power of `freqtrade analysis` comes from the ability to print out the indicator +The real power of `freqtrade backtesting-analysis` comes from the ability to print out the indicator values present on signal candles to allow fine-grained investigation and tuning of buy signal indicators. To print out a column for a given set of indicators, use the `--indicator-list` option: ```bash -freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" --indicator-list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" ``` The indicators have to be present in your strategy's main DataFrame (either for your main From 9a068c0b14ebb80df49d917223469e950ba4a358 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 16:25:31 +0100 Subject: [PATCH 018/162] Add test for each analysis group, remove default table output if not indicator-list --- freqtrade/data/entryexitanalysis.py | 4 +- tests/data/test_entryexitanalysis.py | 149 +++++++++++++++++++++------ 2 files changed, 119 insertions(+), 34 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 15ac6ba09..1c21fcc15 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -173,7 +173,7 @@ def _print_results(analysed_trades, stratname, analysis_groups, exit_reason_list = exit_reason_list.split(",") bigdf = bigdf.loc[(bigdf['exit_reason'].isin(exit_reason_list))] - if indicator_list is not None: + if indicator_list is not None and indicator_list != "": if indicator_list == "all": print(bigdf) else: @@ -183,8 +183,6 @@ def _print_results(analysed_trades, stratname, analysis_groups, available_inds.append(ind) ilist = ["pair", "enter_reason", "exit_reason"] + available_inds _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) - else: - _print_table(bigdf[columns], sortcols=['pair'], show_index=False) else: print("\\_ No trades to show") diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index eadf79179..971cb51aa 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -19,6 +20,8 @@ def entryexitanalysis_cleanup() -> None: def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmpdir, capsys): + caplog.set_level(logging.INFO) + default_conf.update({ "use_exit_signal": True, "exit_profit_only": False, @@ -26,22 +29,32 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp "ignore_roi_if_entry_signal": False, }) patch_exchange(mocker) - result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], - 'profit_ratio': [0.0, 0.0], - 'profit_abs': [0.0, 0.0], + result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC', 'ETH/BTC', 'LTC/BTC'], + 'profit_ratio': [0.025, 0.05, -0.1, -0.05], + 'profit_abs': [0.5, 2.0, -4.0, -2.0], 'open_date': pd.to_datetime(['2018-01-29 18:40:00', - '2018-01-30 03:30:00', ], utc=True + '2018-01-30 03:30:00', + '2018-01-30 08:10:00', + '2018-01-31 13:30:00', ], utc=True ), 'close_date': pd.to_datetime(['2018-01-29 20:45:00', - '2018-01-30 05:35:00', ], utc=True), - 'trade_duration': [235, 40], - 'is_open': [False, False], - 'stake_amount': [0.01, 0.01], - 'open_rate': [0.104445, 0.10302485], - 'close_rate': [0.104969, 0.103541], - "is_short": [False, False], - 'enter_tag': ["enter_tag_long", "enter_tag_long"], - 'exit_reason': [ExitType.ROI, ExitType.ROI] + '2018-01-30 05:35:00', + '2018-01-30 09:10:00', + '2018-01-31 15:00:00', ], utc=True), + 'trade_duration': [235, 40, 60, 90], + 'is_open': [False, False, False, False], + 'stake_amount': [0.01, 0.01, 0.01, 0.01], + 'open_rate': [0.104445, 0.10302485, 0.10302485, 0.10302485], + 'close_rate': [0.104969, 0.103541, 0.102041, 0.102541], + "is_short": [False, False, False, False], + 'enter_tag': ["enter_tag_long_a", + "enter_tag_long_b", + "enter_tag_long_a", + "enter_tag_long_b"], + 'exit_reason': [ExitType.ROI, + ExitType.EXIT_SIGNAL, + ExitType.STOP_LOSS, + ExitType.TRAILING_STOP_LOSS] }) backtestmock = MagicMock(side_effect=[ @@ -85,29 +98,103 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert 'EXIT REASON STATS' in captured.out assert 'LEFT OPEN TRADES REPORT' in captured.out - default_conf.update({ - 'analysis_groups': "0", - 'enter_reason_list': "all", - 'exit_reason_list': "all", - 'indicator_list': "rsi" - }) - - args = [ + base_args = [ 'backtesting-analysis', '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), - '--analysis-groups', '0', - '--indicator-list', 'rsi', - '--strategy', - 'StrategyTestV3Analysis', ] - args = get_args(args) - start_analysis_entries_exits(args) + strat_args = ['--strategy', 'StrategyTestV3Analysis'] + # test group 0 and indicator list + args = get_args(base_args + + ['--analysis-groups', '0', + '--indicator-list', 'close,rsi,profit_abs'] + + strat_args) + start_analysis_entries_exits(args) captured = capsys.readouterr() - assert 'enter_tag_long' in captured.out - assert 'ETH/BTC' in captured.out - assert '34.049' in captured.out assert 'LTC/BTC' in captured.out - assert '54.3204' in captured.out + assert 'ETH/BTC' in captured.out + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'exit_signal' in captured.out + assert 'roi' in captured.out + assert 'stop_loss' in captured.out + assert 'trailing_stop_loss' in captured.out + assert '0.5' in captured.out + assert '-4' in captured.out + assert '-2' in captured.out + assert '-3.5' in captured.out + assert '50' in captured.out + assert '0' in captured.out + assert '0.01616' in captured.out + assert '34.049' in captured.out + assert '0.104104' in captured.out + assert '47.0996' in captured.out + + # test group 1 + args = get_args(base_args + ['--analysis-groups', '1'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'total_profit_pct' in captured.out + assert '-3.5' in captured.out + assert '-1.75' in captured.out + assert '-7.5' in captured.out + assert '-3.75' in captured.out + assert '0' in captured.out + + # test group 2 + args = get_args(base_args + ['--analysis-groups', '2'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'exit_signal' in captured.out + assert 'roi' in captured.out + assert 'stop_loss' in captured.out + assert 'trailing_stop_loss' in captured.out + assert 'total_profit_pct' in captured.out + assert '-10' in captured.out + assert '-5' in captured.out + assert '2.5' in captured.out + + # test group 3 + args = get_args(base_args + ['--analysis-groups', '3'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'LTC/BTC' in captured.out + assert 'ETH/BTC' in captured.out + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'total_profit_pct' in captured.out + assert '-7.5' in captured.out + assert '-3.75' in captured.out + assert '-1.75' in captured.out + assert '0' in captured.out + assert '2' in captured.out + + # test group 4 + args = get_args(base_args + ['--analysis-groups', '4'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'LTC/BTC' in captured.out + assert 'ETH/BTC' in captured.out + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'exit_signal' in captured.out + assert 'roi' in captured.out + assert 'stop_loss' in captured.out + assert 'trailing_stop_loss' in captured.out + assert 'total_profit_pct' in captured.out + assert '-10' in captured.out + assert '-5' in captured.out + assert '-4' in captured.out + assert '0.5' in captured.out + assert '1' in captured.out + assert '2.5' in captured.out From 056047f6352dcbe473890b8ec0df258ea55abdea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 May 2022 20:07:02 +0200 Subject: [PATCH 019/162] Fix --help --- freqtrade/commands/cli_options.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index ce7320b95..e90d3478d 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -616,32 +616,32 @@ AVAILABLE_CLI_OPTIONS = { ), "analysis_groups": Arg( "--analysis-groups", - help=("grouping output - ", - "0: simple wins/losses by enter tag, ", - "1: by enter_tag, ", - "2: by enter_tag and exit_tag, ", - "3: by pair and enter_tag, ", + help=("grouping output - " + "0: simple wins/losses by enter tag, " + "1: by enter_tag, " + "2: by enter_tag and exit_tag, " + "3: by pair and enter_tag, " "4: by pair, enter_ and exit_tag (this can get quite large)"), nargs='?', default="0,1,2", ), "enter_reason_list": Arg( "--enter-reason-list", - help=("Comma separated list of entry signals to analyse. Default: all. ", + help=("Comma separated list of entry signals to analyse. Default: all. " "e.g. 'entry_tag_a,entry_tag_b'"), nargs='?', default='all', ), "exit_reason_list": Arg( "--exit-reason-list", - help=("Comma separated list of exit signals to analyse. Default: all. ", + help=("Comma separated list of exit signals to analyse. Default: all. " "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), nargs='?', default='all', ), "indicator_list": Arg( "--indicator-list", - help=("Comma separated list of indicators to analyse. ", + help=("Comma separated list of indicators to analyse. " "e.g. 'close,rsi,bb_lowerband,profit_abs'"), nargs='?', default='', From c285ad0e2bbd39d2c5e59a38034392b23fb05491 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 May 2022 20:26:24 +0200 Subject: [PATCH 020/162] Remove --strategy parameters, update docs --- docs/utils.md | 50 +++++++++++++++++++++++++++++++++ freqtrade/commands/arguments.py | 15 +++++----- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 9b799e5fc..f87aa2ffc 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -651,6 +651,56 @@ Common arguments: ``` +## Detailed backtest analysis + +Advanced backtest result analysis. + +More details in the [Backtesting analysis](advanced-backtesting.md#analyze-the-buyentry-and-sellexit-tags) Section. + +``` +usage: freqtrade backtesting-analysis [-h] [-v] [--logfile FILE] [-V] + [-c PATH] [-d PATH] [--userdir PATH] + [--analysis-groups [ANALYSIS_GROUPS]] + [--enter-reason-list [ENTER_REASON_LIST]] + [--exit-reason-list [EXIT_REASON_LIST]] + [--indicator-list [INDICATOR_LIST]] + +optional arguments: + -h, --help show this help message and exit + --analysis-groups [ANALYSIS_GROUPS] + grouping output - 0: simple wins/losses by enter tag, + 1: by enter_tag, 2: by enter_tag and exit_tag, 3: by + pair and enter_tag, 4: by pair, enter_ and exit_tag + (this can get quite large) + --enter-reason-list [ENTER_REASON_LIST] + Comma separated list of entry signals to analyse. + Default: all. e.g. 'entry_tag_a,entry_tag_b' + --exit-reason-list [EXIT_REASON_LIST] + Comma separated list of exit signals to analyse. + Default: all. e.g. + 'exit_tag_a,roi,stop_loss,trailing_stop_loss' + --indicator-list [INDICATOR_LIST] + Comma separated list of indicators to analyse. e.g. + 'close,rsi,bb_lowerband,profit_abs' + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + ## List Hyperopt results You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` sub-command. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index d5831a2ac..aed96d042 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -287,6 +287,14 @@ class Arguments: backtesting_show_cmd.set_defaults(func=start_backtesting_show) self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd) + # Add backtesting analysis subcommand + analysis_cmd = subparsers.add_parser('backtesting-analysis', + help='Backtest Analysis module.', + parents=[_common_parser]) + analysis_cmd.set_defaults(func=start_analysis_entries_exits) + self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) + + # Add edge subcommand edge_cmd = subparsers.add_parser('edge', help='Edge module.', parents=[_common_parser, _strategy_parser]) @@ -419,10 +427,3 @@ class Arguments: parents=[_common_parser]) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) - - # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('backtesting-analysis', - help='Backtest Analysis module.', - parents=[_common_parser, _strategy_parser]) - analysis_cmd.set_defaults(func=start_analysis_entries_exits) - self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) From be6e0813db1e3bc45f33381c67b8129cd3404de4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 May 2022 06:34:08 +0200 Subject: [PATCH 021/162] Remove --strategy from analysis test --- freqtrade/commands/arguments.py | 1 - tests/data/test_entryexitanalysis.py | 21 ++++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index aed96d042..6092c630b 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -294,7 +294,6 @@ class Arguments: analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) - # Add edge subcommand edge_cmd = subparsers.add_parser('edge', help='Edge module.', parents=[_common_parser, _strategy_parser]) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 971cb51aa..6209110fe 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -1,5 +1,4 @@ import logging -from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pandas as pd @@ -82,13 +81,10 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), - '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), '--timeframe', '5m', '--timerange', '1515560100-1517287800', '--export', 'signals', '--cache', 'none', - '--strategy-list', - 'StrategyTestV3Analysis', ] args = get_args(args) start_backtesting(args) @@ -104,13 +100,12 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), ] - strat_args = ['--strategy', 'StrategyTestV3Analysis'] # test group 0 and indicator list args = get_args(base_args + ['--analysis-groups', '0', - '--indicator-list', 'close,rsi,profit_abs'] + - strat_args) + '--indicator-list', 'close,rsi,profit_abs'] + ) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out @@ -133,8 +128,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '47.0996' in captured.out # test group 1 - args = get_args(base_args + ['--analysis-groups', '1'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '1']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -147,8 +141,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '0' in captured.out # test group 2 - args = get_args(base_args + ['--analysis-groups', '2'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '2']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -163,8 +156,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2.5' in captured.out # test group 3 - args = get_args(base_args + ['--analysis-groups', '3'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '3']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out @@ -179,8 +171,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2' in captured.out # test group 4 - args = get_args(base_args + ['--analysis-groups', '4'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '4']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out From cce8d1aa4d4f424d35a97c6f7cbb79a698dbd38b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 May 2022 08:48:34 +0000 Subject: [PATCH 022/162] Update get_market_leverage_tiers to be async --- freqtrade/exchange/exchange.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c1a9059a7..c292d7dcb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2131,10 +2131,11 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e - @retrier - def get_market_leverage_tiers(self, symbol) -> List[Dict]: + @retrier_async + async def get_market_leverage_tiers(self, symbol: str) -> Tuple[str, List[Dict]]: try: - return self._api.fetch_market_leverage_tiers(symbol) + tier = await self._api_async.fetch_market_leverage_tiers(symbol) + return symbol, tier except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: @@ -2168,8 +2169,14 @@ class Exchange: f"Initializing leverage_tiers for {len(symbols)} markets. " "This will take about a minute.") - for symbol in sorted(symbols): - tiers[symbol] = self.get_market_leverage_tiers(symbol) + coros = [self.get_market_leverage_tiers(symbol) for symbol in sorted(symbols)] + + for input_coro in chunks(coros, 100): + + results = self.loop.run_until_complete( + asyncio.gather(*input_coro, return_exceptions=True)) + for symbol, res in results: + tiers[symbol] = res logger.info(f"Done initializing {len(symbols)} markets.") From ea537b32c73f6ce0a0cd17c1630292a3f0bcb9c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 May 2022 11:40:14 +0000 Subject: [PATCH 023/162] Update tests for leverage_tier_loading --- tests/conftest.py | 16 ++++++++++++++-- tests/exchange/test_okx.py | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 02738b0e9..c5c253891 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,9 +78,21 @@ def get_args(args): # Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines -def get_mock_coro(return_value): +# TODO: This should be replaced with AsyncMock once support for python 3.7 is dropped. +def get_mock_coro(return_value=None, side_effect=None): async def mock_coro(*args, **kwargs): - return return_value + if side_effect: + if isinstance(side_effect, list): + effect = side_effect.pop(0) + else: + effect = side_effect + if isinstance(effect, Exception): + raise effect + if callable(effect): + return effect(*args, **kwargs) + return effect + else: + return return_value return Mock(wraps=mock_coro) diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 19c09ad9e..91c4a3368 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -6,7 +6,7 @@ import pytest from freqtrade.enums import MarginMode, TradingMode from freqtrade.enums.candletype import CandleType from freqtrade.exchange.exchange import timeframe_to_minutes -from tests.conftest import get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -273,7 +273,7 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets): 'fetchLeverageTiers': False, 'fetchMarketLeverageTiers': True, }) - api_mock.fetch_market_leverage_tiers = MagicMock(side_effect=[ + api_mock.fetch_market_leverage_tiers = get_mock_coro(side_effect=[ [ { 'tier': 1, From afd8e85835c27a051b188296fc078a22bb0af5ef Mon Sep 17 00:00:00 2001 From: Anuj Shah Date: Wed, 1 Jun 2022 15:54:32 +0530 Subject: [PATCH 024/162] feat: add support for discord notification --- freqtrade/freqtradebot.py | 131 +++++++++++++++++++++++++++----------- 1 file changed, 94 insertions(+), 37 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index fba63459b..f7e022987 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -2,6 +2,7 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() """ import copy +import json import logging import traceback from datetime import datetime, time, timezone @@ -9,6 +10,7 @@ from math import isclose from threading import Lock from typing import Any, Dict, List, Optional, Tuple +import requests from schedule import Scheduler from freqtrade import __version__, constants @@ -34,7 +36,6 @@ from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets - logger = logging.getLogger(__name__) @@ -379,9 +380,9 @@ class FreqtradeBot(LoggingMixin): except ExchangeError: logger.warning(f"Error updating {order.order_id}.") -# -# BUY / enter positions / open trades logic and methods -# + # + # BUY / enter positions / open trades logic and methods + # def enter_positions(self) -> int: """ @@ -489,9 +490,9 @@ class FreqtradeBot(LoggingMixin): else: return False -# -# BUY / increase positions / DCA logic and methods -# + # + # BUY / increase positions / DCA logic and methods + # def process_open_trade_positions(self): """ Tries to execute additional buy or sell orders for open trades (positions) @@ -579,16 +580,16 @@ class FreqtradeBot(LoggingMixin): return False def execute_entry( - self, - pair: str, - stake_amount: float, - price: Optional[float] = None, - *, - is_short: bool = False, - ordertype: Optional[str] = None, - enter_tag: Optional[str] = None, - trade: Optional[Trade] = None, - order_adjust: bool = False + self, + pair: str, + stake_amount: float, + price: Optional[float] = None, + *, + is_short: bool = False, + ordertype: Optional[str] = None, + enter_tag: Optional[str] = None, + trade: Optional[Trade] = None, + order_adjust: bool = False ) -> bool: """ Executes a limit buy for the given pair @@ -622,9 +623,9 @@ class FreqtradeBot(LoggingMixin): if not pos_adjust and not strategy_safe_wrapper( self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, - time_in_force=time_in_force, current_time=datetime.now(timezone.utc), - entry_tag=enter_tag, side=trade_side): + pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, + time_in_force=time_in_force, current_time=datetime.now(timezone.utc), + entry_tag=enter_tag, side=trade_side): logger.info(f"User requested abortion of buying {pair}") return False order = self.exchange.create_order( @@ -746,11 +747,11 @@ class FreqtradeBot(LoggingMixin): return trade def get_valid_enter_price_and_stake( - self, pair: str, price: Optional[float], stake_amount: float, - trade_side: LongShort, - entry_tag: Optional[str], - trade: Optional[Trade], - order_adjust: bool, + self, pair: str, price: Optional[float], stake_amount: float, + trade_side: LongShort, + entry_tag: Optional[str], + trade: Optional[Trade], + order_adjust: bool, ) -> Tuple[float, float, float]: if price: @@ -885,9 +886,9 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) -# -# SELL / exit positions / close trades logic and methods -# + # + # SELL / exit positions / close trades logic and methods + # def exit_positions(self, trades: List[Any]) -> int: """ @@ -1059,10 +1060,10 @@ class FreqtradeBot(LoggingMixin): # Finally we check if stoploss on exchange should be moved up because of trailing. # Triggered Orders are now real orders - so don't replace stoploss anymore if ( - trade.is_open and stoploss_order - and stoploss_order.get('status_stop') != 'triggered' - and (self.config.get('trailing_stop', False) - or self.config.get('use_custom_stoploss', False)) + trade.is_open and stoploss_order + and stoploss_order.get('status_stop') != 'triggered' + and (self.config.get('trailing_stop', False) + or self.config.get('use_custom_stoploss', False)) ): # if trailing stoploss is enabled we check if stoploss value has changed # in which case we cancel stoploss order and put another one with new @@ -1145,7 +1146,7 @@ class FreqtradeBot(LoggingMixin): if not_closed: if fully_cancelled or (order_obj and self.strategy.ft_check_timed_out( - trade, order_obj, datetime.now(timezone.utc))): + trade, order_obj, datetime.now(timezone.utc))): self.handle_timedout_order(order, trade) else: self.replace_order(order, order_obj, trade) @@ -1424,7 +1425,7 @@ class FreqtradeBot(LoggingMixin): # if stoploss is on exchange and we are on dry_run mode, # we consider the sell price stop price if (self.config['dry_run'] and exit_type == 'stoploss' - and self.strategy.order_types['stoploss_on_exchange']): + and self.strategy.order_types['stoploss_on_exchange']): limit = trade.stop_loss # set custom_exit_price if available @@ -1543,6 +1544,43 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) + open_date = trade.open_date.strftime('%Y-%m-%d %H:%M:%S') + close_date = trade.close_date.strftime('%Y-%m-%d %H:%M:%S') if trade.close_date else None + + # Send the message to the discord bot + embeds = [{ + 'title': '{} Trade: {}'.format( + 'Profit' if profit_ratio > 0 else 'Loss', + trade.pair), + 'color': (0x00FF00 if profit_ratio > 0 else 0xFF0000), + 'fields': [ + {'name': 'Trade ID', 'value': trade.id, 'inline': True}, + {'name': 'Exchange', 'value': trade.exchange.capitalize(), 'inline': True}, + {'name': 'Pair', 'value': trade.pair, 'inline': True}, + {'name': 'Direction', 'value': 'Short' if trade.is_short else 'Long', 'inline': True}, + {'name': 'Open rate', 'value': trade.open_rate, 'inline': True}, + {'name': 'Close rate', 'value': trade.close_rate, 'inline': True}, + {'name': 'Amount', 'value': trade.amount, 'inline': True}, + {'name': 'Open order', 'value': trade.open_order_id, 'inline': True}, + {'name': 'Open date', 'value': open_date, 'inline': True}, + {'name': 'Close date', 'value': close_date, 'inline': True}, + {'name': 'Profit', 'value': profit_trade, 'inline': True}, + {'name': 'Profitability', 'value': '{:.2f}%'.format(profit_ratio * 100), 'inline': True}, + {'name': 'Stake currency', 'value': self.config['stake_currency'], 'inline': True}, + {'name': 'Fiat currency', 'value': self.config.get('fiat_display_currency', None), 'inline': True}, + {'name': 'Buy Tag', 'value': trade.enter_tag, 'inline': True}, + {'name': 'Sell Reason', 'value': trade.exit_reason, 'inline': True}, + {'name': 'Strategy', 'value': trade.strategy, 'inline': True}, + {'name': 'Timeframe', 'value': trade.timeframe, 'inline': True}, + ], + }] + # convert all value in fields to string + for embed in embeds: + for field in embed['fields']: + field['value'] = str(field['value']) + if fill: + self.discord_send(embeds) + def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a sell cancel occurred. @@ -1593,9 +1631,9 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) -# -# Common update trade state methods -# + # + # Common update trade state methods + # def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None, stoploss_order: bool = False, send_msg: bool = True) -> bool: @@ -1818,3 +1856,22 @@ class FreqtradeBot(LoggingMixin): return max( min(valid_custom_price, max_custom_price_allowed), min_custom_price_allowed) + + def discord_send(self, embeds): + if not 'discord' in self.config or self.config['discord']['enabled'] == False: + return + if self.config['runmode'].value in ('dry_run', 'live'): + webhook_url = self.config['discord']['webhook_url'] + + payload = { + "embeds": embeds + } + + headers = { + "Content-Type": "application/json" + } + + try: + requests.post(webhook_url, data=json.dumps(payload), headers=headers) + except Exception as e: + logger.error(f"Error sending discord message: {e}") From 45c47bda6000b2b57026fdedffaaa69f8fc1797e Mon Sep 17 00:00:00 2001 From: Anuj Shah Date: Wed, 1 Jun 2022 21:14:48 +0530 Subject: [PATCH 025/162] refactor into discord rpc module --- freqtrade/freqtradebot.py | 131 ++++++++++------------------------- freqtrade/rpc/discord.py | 101 +++++++++++++++++++++++++++ freqtrade/rpc/rpc_manager.py | 6 ++ 3 files changed, 144 insertions(+), 94 deletions(-) create mode 100644 freqtrade/rpc/discord.py diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f7e022987..fba63459b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -2,7 +2,6 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() """ import copy -import json import logging import traceback from datetime import datetime, time, timezone @@ -10,7 +9,6 @@ from math import isclose from threading import Lock from typing import Any, Dict, List, Optional, Tuple -import requests from schedule import Scheduler from freqtrade import __version__, constants @@ -36,6 +34,7 @@ from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets + logger = logging.getLogger(__name__) @@ -380,9 +379,9 @@ class FreqtradeBot(LoggingMixin): except ExchangeError: logger.warning(f"Error updating {order.order_id}.") - # - # BUY / enter positions / open trades logic and methods - # +# +# BUY / enter positions / open trades logic and methods +# def enter_positions(self) -> int: """ @@ -490,9 +489,9 @@ class FreqtradeBot(LoggingMixin): else: return False - # - # BUY / increase positions / DCA logic and methods - # +# +# BUY / increase positions / DCA logic and methods +# def process_open_trade_positions(self): """ Tries to execute additional buy or sell orders for open trades (positions) @@ -580,16 +579,16 @@ class FreqtradeBot(LoggingMixin): return False def execute_entry( - self, - pair: str, - stake_amount: float, - price: Optional[float] = None, - *, - is_short: bool = False, - ordertype: Optional[str] = None, - enter_tag: Optional[str] = None, - trade: Optional[Trade] = None, - order_adjust: bool = False + self, + pair: str, + stake_amount: float, + price: Optional[float] = None, + *, + is_short: bool = False, + ordertype: Optional[str] = None, + enter_tag: Optional[str] = None, + trade: Optional[Trade] = None, + order_adjust: bool = False ) -> bool: """ Executes a limit buy for the given pair @@ -623,9 +622,9 @@ class FreqtradeBot(LoggingMixin): if not pos_adjust and not strategy_safe_wrapper( self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, - time_in_force=time_in_force, current_time=datetime.now(timezone.utc), - entry_tag=enter_tag, side=trade_side): + pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, + time_in_force=time_in_force, current_time=datetime.now(timezone.utc), + entry_tag=enter_tag, side=trade_side): logger.info(f"User requested abortion of buying {pair}") return False order = self.exchange.create_order( @@ -747,11 +746,11 @@ class FreqtradeBot(LoggingMixin): return trade def get_valid_enter_price_and_stake( - self, pair: str, price: Optional[float], stake_amount: float, - trade_side: LongShort, - entry_tag: Optional[str], - trade: Optional[Trade], - order_adjust: bool, + self, pair: str, price: Optional[float], stake_amount: float, + trade_side: LongShort, + entry_tag: Optional[str], + trade: Optional[Trade], + order_adjust: bool, ) -> Tuple[float, float, float]: if price: @@ -886,9 +885,9 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - # - # SELL / exit positions / close trades logic and methods - # +# +# SELL / exit positions / close trades logic and methods +# def exit_positions(self, trades: List[Any]) -> int: """ @@ -1060,10 +1059,10 @@ class FreqtradeBot(LoggingMixin): # Finally we check if stoploss on exchange should be moved up because of trailing. # Triggered Orders are now real orders - so don't replace stoploss anymore if ( - trade.is_open and stoploss_order - and stoploss_order.get('status_stop') != 'triggered' - and (self.config.get('trailing_stop', False) - or self.config.get('use_custom_stoploss', False)) + trade.is_open and stoploss_order + and stoploss_order.get('status_stop') != 'triggered' + and (self.config.get('trailing_stop', False) + or self.config.get('use_custom_stoploss', False)) ): # if trailing stoploss is enabled we check if stoploss value has changed # in which case we cancel stoploss order and put another one with new @@ -1146,7 +1145,7 @@ class FreqtradeBot(LoggingMixin): if not_closed: if fully_cancelled or (order_obj and self.strategy.ft_check_timed_out( - trade, order_obj, datetime.now(timezone.utc))): + trade, order_obj, datetime.now(timezone.utc))): self.handle_timedout_order(order, trade) else: self.replace_order(order, order_obj, trade) @@ -1425,7 +1424,7 @@ class FreqtradeBot(LoggingMixin): # if stoploss is on exchange and we are on dry_run mode, # we consider the sell price stop price if (self.config['dry_run'] and exit_type == 'stoploss' - and self.strategy.order_types['stoploss_on_exchange']): + and self.strategy.order_types['stoploss_on_exchange']): limit = trade.stop_loss # set custom_exit_price if available @@ -1544,43 +1543,6 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - open_date = trade.open_date.strftime('%Y-%m-%d %H:%M:%S') - close_date = trade.close_date.strftime('%Y-%m-%d %H:%M:%S') if trade.close_date else None - - # Send the message to the discord bot - embeds = [{ - 'title': '{} Trade: {}'.format( - 'Profit' if profit_ratio > 0 else 'Loss', - trade.pair), - 'color': (0x00FF00 if profit_ratio > 0 else 0xFF0000), - 'fields': [ - {'name': 'Trade ID', 'value': trade.id, 'inline': True}, - {'name': 'Exchange', 'value': trade.exchange.capitalize(), 'inline': True}, - {'name': 'Pair', 'value': trade.pair, 'inline': True}, - {'name': 'Direction', 'value': 'Short' if trade.is_short else 'Long', 'inline': True}, - {'name': 'Open rate', 'value': trade.open_rate, 'inline': True}, - {'name': 'Close rate', 'value': trade.close_rate, 'inline': True}, - {'name': 'Amount', 'value': trade.amount, 'inline': True}, - {'name': 'Open order', 'value': trade.open_order_id, 'inline': True}, - {'name': 'Open date', 'value': open_date, 'inline': True}, - {'name': 'Close date', 'value': close_date, 'inline': True}, - {'name': 'Profit', 'value': profit_trade, 'inline': True}, - {'name': 'Profitability', 'value': '{:.2f}%'.format(profit_ratio * 100), 'inline': True}, - {'name': 'Stake currency', 'value': self.config['stake_currency'], 'inline': True}, - {'name': 'Fiat currency', 'value': self.config.get('fiat_display_currency', None), 'inline': True}, - {'name': 'Buy Tag', 'value': trade.enter_tag, 'inline': True}, - {'name': 'Sell Reason', 'value': trade.exit_reason, 'inline': True}, - {'name': 'Strategy', 'value': trade.strategy, 'inline': True}, - {'name': 'Timeframe', 'value': trade.timeframe, 'inline': True}, - ], - }] - # convert all value in fields to string - for embed in embeds: - for field in embed['fields']: - field['value'] = str(field['value']) - if fill: - self.discord_send(embeds) - def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a sell cancel occurred. @@ -1631,9 +1593,9 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - # - # Common update trade state methods - # +# +# Common update trade state methods +# def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None, stoploss_order: bool = False, send_msg: bool = True) -> bool: @@ -1856,22 +1818,3 @@ class FreqtradeBot(LoggingMixin): return max( min(valid_custom_price, max_custom_price_allowed), min_custom_price_allowed) - - def discord_send(self, embeds): - if not 'discord' in self.config or self.config['discord']['enabled'] == False: - return - if self.config['runmode'].value in ('dry_run', 'live'): - webhook_url = self.config['discord']['webhook_url'] - - payload = { - "embeds": embeds - } - - headers = { - "Content-Type": "application/json" - } - - try: - requests.post(webhook_url, data=json.dumps(payload), headers=headers) - except Exception as e: - logger.error(f"Error sending discord message: {e}") diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py new file mode 100644 index 000000000..ee9970dc5 --- /dev/null +++ b/freqtrade/rpc/discord.py @@ -0,0 +1,101 @@ +import json +import logging +from typing import Dict, Any + +import requests + +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPCHandler, RPC + + +class Discord(RPCHandler): + def __init__(self, rpc: 'RPC', config: Dict[str, Any]): + super().__init__(rpc, config) + self.logger = logging.getLogger(__name__) + self.strategy = config.get('strategy', '') + self.timeframe = config.get('timeframe', '') + self.config = config + + def send_msg(self, msg: Dict[str, str]) -> None: + self._send_msg(msg) + + def _send_msg(self, msg): + """ + msg = { + 'type': (RPCMessageType.EXIT_FILL if fill + else RPCMessageType.EXIT), + 'trade_id': trade.id, + 'exchange': trade.exchange.capitalize(), + 'pair': trade.pair, + 'leverage': trade.leverage, + 'direction': 'Short' if trade.is_short else 'Long', + 'gain': gain, + 'limit': profit_rate, + 'order_type': order_type, + 'amount': trade.amount, + 'open_rate': trade.open_rate, + 'close_rate': trade.close_rate, + 'current_rate': current_rate, + 'profit_amount': profit_trade, + 'profit_ratio': profit_ratio, + 'buy_tag': trade.enter_tag, + 'enter_tag': trade.enter_tag, + 'sell_reason': trade.exit_reason, # Deprecated + 'exit_reason': trade.exit_reason, + 'open_date': trade.open_date, + 'close_date': trade.close_date or datetime.utcnow(), + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), + } + """ + self.logger.info(f"Sending discord message: {msg}") + + # TODO: handle other message types + if msg['type'] == RPCMessageType.EXIT_FILL: + profit_ratio = msg.get('profit_ratio') + open_date = msg.get('open_date').strftime('%Y-%m-%d %H:%M:%S') + close_date = msg.get('close_date').strftime('%Y-%m-%d %H:%M:%S') if msg.get('close_date') else '' + + embeds = [{ + 'title': '{} Trade: {}'.format( + 'Profit' if profit_ratio > 0 else 'Loss', + msg.get('pair')), + 'color': (0x00FF00 if profit_ratio > 0 else 0xFF0000), + 'fields': [ + {'name': 'Trade ID', 'value': msg.get('id'), 'inline': True}, + {'name': 'Exchange', 'value': msg.get('exchange').capitalize(), 'inline': True}, + {'name': 'Pair', 'value': msg.get('pair'), 'inline': True}, + {'name': 'Direction', 'value': 'Short' if msg.get('is_short') else 'Long', 'inline': True}, + {'name': 'Open rate', 'value': msg.get('open_rate'), 'inline': True}, + {'name': 'Close rate', 'value': msg.get('close_rate'), 'inline': True}, + {'name': 'Amount', 'value': msg.get('amount'), 'inline': True}, + {'name': 'Open order', 'value': msg.get('open_order_id'), 'inline': True}, + {'name': 'Open date', 'value': open_date, 'inline': True}, + {'name': 'Close date', 'value': close_date, 'inline': True}, + {'name': 'Profit', 'value': msg.get('profit_amount'), 'inline': True}, + {'name': 'Profitability', 'value': '{:.2f}%'.format(profit_ratio * 100), 'inline': True}, + {'name': 'Stake currency', 'value': msg.get('stake_currency'), 'inline': True}, + {'name': 'Fiat currency', 'value': msg.get('fiat_display_currency'), 'inline': True}, + {'name': 'Buy Tag', 'value': msg.get('enter_tag'), 'inline': True}, + {'name': 'Sell Reason', 'value': msg.get('exit_reason'), 'inline': True}, + {'name': 'Strategy', 'value': self.strategy, 'inline': True}, + {'name': 'Timeframe', 'value': self.timeframe, 'inline': True}, + ], + }] + + # convert all value in fields to string for discord + for embed in embeds: + for field in embed['fields']: + field['value'] = str(field['value']) + + # Send the message to discord channel + payload = { + 'embeds': embeds, + } + headers = { + 'Content-Type': 'application/json', + } + try: + requests.post(self.config['discord']['webhook_url'], data=json.dumps(payload), headers=headers) + except Exception as e: + self.logger.error(f"Failed to send discord message: {e}") diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index d97d1df5f..66e84029f 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -27,6 +27,12 @@ class RPCManager: from freqtrade.rpc.telegram import Telegram self.registered_modules.append(Telegram(self._rpc, config)) + # Enable discord + if config.get('discord', {}).get('enabled', False): + logger.info('Enabling rpc.discord ...') + from freqtrade.rpc.discord import Discord + self.registered_modules.append(Discord(self._rpc, config)) + # Enable Webhook if config.get('webhook', {}).get('enabled', False): logger.info('Enabling rpc.webhook ...') From eb4adeab4d7511fe084924e72c14065c6c106ebf Mon Sep 17 00:00:00 2001 From: Anuj Shah Date: Thu, 2 Jun 2022 11:19:29 +0530 Subject: [PATCH 026/162] fix flake8 issues --- freqtrade/rpc/discord.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index ee9970dc5..43a8e9a05 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -54,7 +54,8 @@ class Discord(RPCHandler): if msg['type'] == RPCMessageType.EXIT_FILL: profit_ratio = msg.get('profit_ratio') open_date = msg.get('open_date').strftime('%Y-%m-%d %H:%M:%S') - close_date = msg.get('close_date').strftime('%Y-%m-%d %H:%M:%S') if msg.get('close_date') else '' + close_date = msg.get('close_date').strftime( + '%Y-%m-%d %H:%M:%S') if msg.get('close_date') else '' embeds = [{ 'title': '{} Trade: {}'.format( @@ -65,7 +66,8 @@ class Discord(RPCHandler): {'name': 'Trade ID', 'value': msg.get('id'), 'inline': True}, {'name': 'Exchange', 'value': msg.get('exchange').capitalize(), 'inline': True}, {'name': 'Pair', 'value': msg.get('pair'), 'inline': True}, - {'name': 'Direction', 'value': 'Short' if msg.get('is_short') else 'Long', 'inline': True}, + {'name': 'Direction', 'value': 'Short' if msg.get( + 'is_short') else 'Long', 'inline': True}, {'name': 'Open rate', 'value': msg.get('open_rate'), 'inline': True}, {'name': 'Close rate', 'value': msg.get('close_rate'), 'inline': True}, {'name': 'Amount', 'value': msg.get('amount'), 'inline': True}, @@ -73,9 +75,11 @@ class Discord(RPCHandler): {'name': 'Open date', 'value': open_date, 'inline': True}, {'name': 'Close date', 'value': close_date, 'inline': True}, {'name': 'Profit', 'value': msg.get('profit_amount'), 'inline': True}, - {'name': 'Profitability', 'value': '{:.2f}%'.format(profit_ratio * 100), 'inline': True}, + {'name': 'Profitability', 'value': '{:.2f}%'.format( + profit_ratio * 100), 'inline': True}, {'name': 'Stake currency', 'value': msg.get('stake_currency'), 'inline': True}, - {'name': 'Fiat currency', 'value': msg.get('fiat_display_currency'), 'inline': True}, + {'name': 'Fiat currency', 'value': msg.get( + 'fiat_display_currency'), 'inline': True}, {'name': 'Buy Tag', 'value': msg.get('enter_tag'), 'inline': True}, {'name': 'Sell Reason', 'value': msg.get('exit_reason'), 'inline': True}, {'name': 'Strategy', 'value': self.strategy, 'inline': True}, @@ -96,6 +100,9 @@ class Discord(RPCHandler): 'Content-Type': 'application/json', } try: - requests.post(self.config['discord']['webhook_url'], data=json.dumps(payload), headers=headers) + requests.post( + self.config['discord']['webhook_url'], + data=json.dumps(payload), + headers=headers) except Exception as e: self.logger.error(f"Failed to send discord message: {e}") From 27bea580d492afa73f2b79e7ad3f63f8995fa4ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jun 2022 09:40:04 +0200 Subject: [PATCH 027/162] Fix rest-client script's force_enter closes #6927 --- scripts/rest_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ecbb65253..e5d358c98 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -261,7 +261,7 @@ class FtRestClient(): } return self._post("forcebuy", data=data) - def force_enter(self, pair, side, price=None): + def forceenter(self, pair, side, price=None): """Force entering a trade :param pair: Pair to buy (ETH/BTC) @@ -273,7 +273,7 @@ class FtRestClient(): "side": side, "price": price, } - return self._post("force_enter", data=data) + return self._post("forceenter", data=data) def forceexit(self, tradeid): """Force-exit a trade. From a790bad1e4dd8248d95c9ed8d2d9d50a76a196b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jun 2022 10:21:06 +0200 Subject: [PATCH 028/162] Add entry_tag to leverage callback closes #6929 --- docs/strategy-callbacks.md | 7 ++++--- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/strategy/interface.py | 5 +++-- .../templates/subtemplates/strategy_methods_advanced.j2 | 5 +++-- tests/strategy/strats/strategy_test_v3.py | 4 ++-- tests/strategy/test_interface.py | 2 ++ 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index f0f7d8f69..656f206a4 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -804,17 +804,18 @@ For markets / exchanges that don't support leverage, this method is ignored. ``` python class AwesomeStrategy(IStrategy): - def leverage(self, pair: str, current_time: 'datetime', current_rate: float, - proposed_leverage: float, max_leverage: float, side: str, + def leverage(self, pair: str, current_time: datetime, current_rate: float, + proposed_leverage: float, max_leverage: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ - Customize leverage for each new trade. + Customize leverage for each new trade. This method is only called in futures mode. :param pair: Pair that's currently analyzed :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in exit_pricing. :param proposed_leverage: A leverage proposed by the bot. :param max_leverage: Max leverage allowed on this pair + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. :param side: 'long' or 'short' - indicating the direction of the proposed trade :return: A leverage amount, which is between 1.0 and max_leverage. """ diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index fba63459b..95eb911cf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -781,7 +781,7 @@ class FreqtradeBot(LoggingMixin): current_rate=enter_limit_requested, proposed_leverage=1.0, max_leverage=max_leverage, - side=trade_side, + side=trade_side, entry_tag=entry_tag, ) if self.trading_mode != TradingMode.SPOT else 1.0 # Cap leverage between 1.0 and max_leverage. leverage = min(max(leverage, 1.0), max_leverage) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fa5065370..aebaecaca 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -704,7 +704,7 @@ class Backtesting: current_rate=row[OPEN_IDX], proposed_leverage=1.0, max_leverage=max_leverage, - side=direction, + side=direction, entry_tag=entry_tag, ) if self._can_short else 1.0 # Cap leverage between 1.0 and max_leverage. leverage = min(max(leverage, 1.0), max_leverage) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 99dd1bfd7..3b3d326ff 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -509,8 +509,8 @@ class IStrategy(ABC, HyperStrategyMixin): return current_order_rate def leverage(self, pair: str, current_time: datetime, current_rate: float, - proposed_leverage: float, max_leverage: float, side: str, - **kwargs) -> float: + proposed_leverage: float, max_leverage: float, entry_tag: Optional[str], + side: str, **kwargs) -> float: """ Customize leverage for each new trade. This method is only called in futures mode. @@ -519,6 +519,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_rate: Rate, calculated based on pricing settings in exit_pricing. :param proposed_leverage: A leverage proposed by the bot. :param max_leverage: Max leverage allowed on this pair + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. :param side: 'long' or 'short' - indicating the direction of the proposed trade :return: A leverage amount, which is between 1.0 and max_leverage. """ diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 103541efe..acefd0363 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -267,8 +267,8 @@ def adjust_trade_position(self, trade: 'Trade', current_time: 'datetime', return None def leverage(self, pair: str, current_time: datetime, current_rate: float, - proposed_leverage: float, max_leverage: float, side: str, - **kwargs) -> float: + proposed_leverage: float, max_leverage: float, entry_tag: Optional[str], + side: str, **kwargs) -> float: """ Customize leverage for each new trade. This method is only called in futures mode. @@ -277,6 +277,7 @@ def leverage(self, pair: str, current_time: datetime, current_rate: float, :param current_rate: Rate, calculated based on pricing settings in exit_pricing. :param proposed_leverage: A leverage proposed by the bot. :param max_leverage: Max leverage allowed on this pair + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. :param side: 'long' or 'short' - indicating the direction of the proposed trade :return: A leverage amount, which is between 1.0 and max_leverage. """ diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 340001ef2..2c7ccbdf2 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -178,8 +178,8 @@ class StrategyTestV3(IStrategy): return dataframe def leverage(self, pair: str, current_time: datetime, current_rate: float, - proposed_leverage: float, max_leverage: float, side: str, - **kwargs) -> float: + proposed_leverage: float, max_leverage: float, entry_tag: Optional[str], + side: str, **kwargs) -> float: # Return 3.0 in all cases. # Bot-logic must make sure it's an allowed leverage and eventually adjust accordingly. diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index e3c0bcfcb..b7b73bdcf 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -615,6 +615,7 @@ def test_leverage_callback(default_conf, side) -> None: proposed_leverage=1.0, max_leverage=5.0, side=side, + entry_tag=None, ) == 1 default_conf['strategy'] = CURRENT_TEST_STRATEGY @@ -626,6 +627,7 @@ def test_leverage_callback(default_conf, side) -> None: proposed_leverage=1.0, max_leverage=5.0, side=side, + entry_tag='entry_tag_test', ) == 3 From c499bb051f4753f20c9daef9660932c2b610ecd7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jun 2022 19:41:17 +0200 Subject: [PATCH 029/162] Allow empty unfilledtimeout for webserver mode --- freqtrade/rpc/api_server/api_schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index f21334bc6..a31c74c2e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -166,7 +166,7 @@ class ShowConfig(BaseModel): trailing_stop_positive: Optional[float] trailing_stop_positive_offset: Optional[float] trailing_only_offset_is_reached: Optional[bool] - unfilledtimeout: UnfilledTimeout + unfilledtimeout: Optional[UnfilledTimeout] # Empty in webserver mode order_types: Optional[OrderTypes] use_custom_stoploss: Optional[bool] timeframe: Optional[str] From f709222943fcc2807561ae4a8fc7bcb9a8d6c66c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 06:53:30 +0200 Subject: [PATCH 030/162] Properly close out orders in backtesting --- freqtrade/optimize/backtesting.py | 1 + freqtrade/persistence/trade_model.py | 1 + 2 files changed, 2 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index aebaecaca..8fe5f509e 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1094,6 +1094,7 @@ class Backtesting: # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) if order and self._get_order_filled(order.price, row): + order.close_bt_order(current_time, trade) trade.open_order_id = None trade.close_date = current_time trade.close(order.price, show_msg=False) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 45a16bfbd..7b475d618 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -166,6 +166,7 @@ class Order(_DECL_BASE): def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): self.order_filled_date = close_date self.filled = self.amount + self.remaining = 0 self.status = 'closed' self.ft_is_open = False if (self.ft_order_side == trade.entry_side From c0ff554d5be871098cd10424fdd579322b5370df Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 May 2022 20:12:05 +0200 Subject: [PATCH 031/162] Cleanup old, left open dry-run orders --- freqtrade/persistence/migrations.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 53e35d9da..b0fdf0412 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -247,6 +247,35 @@ def set_sqlite_to_wal(engine): connection.execute(text("PRAGMA journal_mode=wal")) +def fix_old_dry_orders(engine): + with engine.begin() as connection: + connection.execute( + text( + """ + update orders + set ft_is_open = 0 + where ft_is_open = 1 and (ft_trade_id, order_id) not in ( + select id, stoploss_order_id from trades where stoploss_order_id is not null + ) and ft_order_side = 'stoploss' + and order_id like 'dry_%' + """ + ) + ) + connection.execute( + text( + """ + update orders + set ft_is_open = 0 + where ft_is_open = 1 + and (ft_trade_id, order_id) not in ( + select id, open_order_id from trades where open_order_id is not null + ) and ft_order_side != 'stoploss' + and order_id like 'dry_%' + """ + ) + ) + + def check_migrate(engine, decl_base, previous_tables) -> None: """ Checks if migration is necessary and migrates if necessary @@ -288,3 +317,4 @@ def check_migrate(engine, decl_base, previous_tables) -> None: "start with a fresh database.") set_sqlite_to_wal(engine) + fix_old_dry_orders(engine) From 8369d5bedd25f7679b060fd075be2eb061623ebe Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 May 2022 20:31:45 +0200 Subject: [PATCH 032/162] Include open orders in json responses --- freqtrade/persistence/trade_model.py | 17 ++++++++++++++++- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 7b475d618..ded616f8a 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -395,7 +395,7 @@ class LocalTrade(): ) def to_json(self) -> Dict[str, Any]: - filled_orders = self.select_filled_orders() + filled_orders = self.select_filled_or_open_orders() orders = [order.to_json(self.entry_side) for order in filled_orders] return { @@ -898,6 +898,21 @@ class LocalTrade(): (o.filled or 0) > 0 and o.status in NON_OPEN_EXCHANGE_STATES] + def select_filled_or_open_orders(self) -> List['Order']: + """ + Finds filled or open orders + :param order_side: Side of the order (either 'buy', 'sell', or None) + :return: array of Order objects + """ + return [o for o in self.orders if + ( + o.ft_is_open is False + and (o.filled or 0) > 0 + and o.status in NON_OPEN_EXCHANGE_STATES + ) + or (o.ft_is_open is True and o.status is not None) + ] + @property def nr_of_successful_entries(self) -> int: """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4a274002e..e456b1eef 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -396,7 +396,7 @@ class Telegram(RPCHandler): first_avg = filled_orders[0]["safe_price"] for x, order in enumerate(filled_orders): - if not order['ft_is_entry']: + if not order['ft_is_entry'] or order['is_open'] is True: continue cur_entry_datetime = arrow.get(order["order_filled_date"]) cur_entry_amount = order["amount"] From 79107fd062e9e60f78c467367b7c34cc68f5b6c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 May 2022 07:11:43 +0200 Subject: [PATCH 033/162] Add minimal order object serialization --- freqtrade/data/btanalysis.py | 2 +- freqtrade/persistence/trade_model.py | 44 +++++++++++++++------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index fef432576..0b466241f 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -337,7 +337,7 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame: :param trades: List of trade objects :return: Dataframe with BT_DATA_COLUMNS """ - df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS) + df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS) if len(df) > 0: df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True) df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ded616f8a..0be9d22c1 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -137,31 +137,35 @@ class Order(_DECL_BASE): 'info': {}, } - def to_json(self, entry_side: str) -> Dict[str, Any]: - return { - 'pair': self.ft_pair, - 'order_id': self.order_id, - 'status': self.status, + def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]: + resp = { 'amount': self.amount, - 'average': round(self.average, 8) if self.average else 0, 'safe_price': self.safe_price, - 'cost': self.cost if self.cost else 0, - 'filled': self.filled, 'ft_order_side': self.ft_order_side, - 'is_open': self.ft_is_open, - 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_date else None, - 'order_timestamp': int(self.order_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, - 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_filled_date else None, 'order_filled_timestamp': int(self.order_filled_date.replace( tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None, - 'order_type': self.order_type, - 'price': self.price, 'ft_is_entry': self.ft_order_side == entry_side, - 'remaining': self.remaining, } + if not minified: + resp.update({ + 'pair': self.ft_pair, + 'order_id': self.order_id, + 'status': self.status, + 'average': round(self.average, 8) if self.average else 0, + 'cost': self.cost if self.cost else 0, + 'filled': self.filled, + 'is_open': self.ft_is_open, + 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_date else None, + 'order_timestamp': int(self.order_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, + 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_filled_date else None, + 'order_type': self.order_type, + 'price': self.price, + 'remaining': self.remaining, + }) + return resp def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): self.order_filled_date = close_date @@ -394,9 +398,9 @@ class LocalTrade(): f'open_rate={self.open_rate:.8f}, open_since={open_since})' ) - def to_json(self) -> Dict[str, Any]: + def to_json(self, minified: bool = False) -> Dict[str, Any]: filled_orders = self.select_filled_or_open_orders() - orders = [order.to_json(self.entry_side) for order in filled_orders] + orders = [order.to_json(self.entry_side, minified) for order in filled_orders] return { 'trade_id': self.id, From 786bc3616352a650d4107a539aad84f7c32d1714 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:44 +0000 Subject: [PATCH 034/162] Bump orjson from 3.6.8 to 3.7.1 Bumps [orjson](https://github.com/ijl/orjson) from 3.6.8 to 3.7.1. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.6.8...3.7.1) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..21d80571f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.6 # Properly format api responses -orjson==3.6.8 +orjson==3.7.1 # Notify systemd sdnotify==0.3.2 From 04cb49b7e404fb0ab29245e769296f7c5ec17d41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:48 +0000 Subject: [PATCH 035/162] Bump filelock from 3.7.0 to 3.7.1 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.7.0 to 3.7.1. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.7.0...3.7.1) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b8762214a..94e59ec15 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,5 +5,5 @@ scipy==1.8.1 scikit-learn==1.1.1 scikit-optimize==0.9.0 -filelock==3.7.0 +filelock==3.7.1 progressbar2==4.0.0 From 6547f3aadb96f60e9d5a42b0008a55de1c47e75c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:52 +0000 Subject: [PATCH 036/162] Bump mkdocs-material from 8.2.16 to 8.3.2 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.2.16 to 8.3.2. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.2.16...8.3.2) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index e7ca17c34..f351151ab 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.2.16 +mkdocs-material==8.3.2 mdx_truly_sane_lists==1.2 pymdown-extensions==9.4 jinja2==3.1.2 From 35316ec06841b9b2638ab6068c6d58aa3c15991a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:55 +0000 Subject: [PATCH 037/162] Bump jsonschema from 4.5.1 to 4.6.0 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.5.1 to 4.6.0. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.5.1...v4.6.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..d0b662d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.2 cachetools==4.2.2 requests==2.27.1 urllib3==1.26.9 -jsonschema==4.5.1 +jsonschema==4.6.0 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.9 From 963dc0221caa59b8cf4cb2309cab6735f8be6161 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:59 +0000 Subject: [PATCH 038/162] Bump types-requests from 2.27.29 to 2.27.30 Bumps [types-requests](https://github.com/python/typeshed) from 2.27.29 to 2.27.30. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6a7e15870..4eb157aae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,6 +24,6 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.0.1 types-filelock==3.2.6 -types-requests==2.27.29 +types-requests==2.27.30 types-tabulate==0.8.9 types-python-dateutil==2.8.17 From 4affa75ff5103e95fdbc6c59fa457423326fdc74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:02:07 +0000 Subject: [PATCH 039/162] Bump sqlalchemy from 1.4.36 to 1.4.37 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.36 to 1.4.37. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..717577480 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.84.39 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 -SQLAlchemy==1.4.36 +SQLAlchemy==1.4.37 python-telegram-bot==13.12 arrow==1.2.2 cachetools==4.2.2 From 05922e9ebc0c7911f0bc75e50b5a57dc6a0cc29d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:02:15 +0000 Subject: [PATCH 040/162] Bump ccxt from 1.84.39 to 1.84.97 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.84.39 to 1.84.97. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.84.39...1.84.97) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..432ff976d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.84.39 +ccxt==1.84.97 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 99f6c75c40dc95073ec81a03c82e775d87753667 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 10:22:19 +0200 Subject: [PATCH 041/162] Bump types-requests precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 95a1d5002..685d789ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.0.1 - types-filelock==3.2.6 - - types-requests==2.27.29 + - types-requests==2.27.30 - types-tabulate==0.8.9 - types-python-dateutil==2.8.17 # stages: [push] From ea9b68baddeb76f2581660d13ff11b797f4a6b00 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 10:50:48 +0200 Subject: [PATCH 042/162] Add updating freqtrade to updating desc --- docs/updating.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/updating.md b/docs/updating.md index 1839edc4c..8dc7279a4 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -32,4 +32,8 @@ Please ensure that you're also updating dependencies - otherwise things might br ``` bash git pull pip install -U -r requirements.txt +pip install -e . + +# Ensure freqUI is at the latest version +freqtrade install-ui ``` From 82c5a6b29dc1c45e0e542d2caace0fb2d87dad68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 10:57:33 +0200 Subject: [PATCH 043/162] Update CI to use concurrency --- .github/workflows/ci.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2e420e8e..c3ed6d80d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ on: schedule: - cron: '0 5 * * 4' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_linux: @@ -296,17 +300,17 @@ jobs: details: Freqtrade doc test failed! webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} - cleanup-prior-runs: - permissions: - actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it - contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch - runs-on: ubuntu-20.04 - steps: - - name: Cleanup previous runs on this branch - uses: rokroskar/workflow-run-cleanup-action@v0.3.3 - if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + # cleanup-prior-runs: + # permissions: + # actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it + # contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch + # runs-on: ubuntu-20.04 + # steps: + # - name: Cleanup previous runs on this branch + # uses: rokroskar/workflow-run-cleanup-action@v0.3.3 + # if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" + # env: + # GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # Notify only once - when CI completes (and after deploy) in case it's successfull notify-complete: From 0b806af48756bcb5190fadc1cd50cd9b2ff32b3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 May 2022 07:17:22 +0200 Subject: [PATCH 044/162] Add orders column to btresult --- freqtrade/data/btanalysis.py | 4 +++- freqtrade/optimize/optimize_reports.py | 4 ---- tests/data/test_btanalysis.py | 4 ++-- tests/optimize/test_backtesting.py | 17 +++++++++++++++++ .../test_backtesting_adjust_position.py | 7 ++++++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0b466241f..9e38f6833 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'profit_ratio', 'profit_abs', 'exit_reason', 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag', - 'is_short' + 'is_short', 'open_timestamp', 'close_timestamp', 'orders' ] @@ -283,6 +283,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non if 'enter_tag' not in df.columns: df['enter_tag'] = df['buy_tag'] df = df.drop(['buy_tag'], axis=1) + if 'orders' not in df.columns: + df.loc[:, 'orders'] = None else: # old format - only with lists. diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 93336fa3f..e3dd17411 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Union -from numpy import int64 from pandas import DataFrame, to_datetime from tabulate import tabulate @@ -417,9 +416,6 @@ def generate_strategy_stats(pairlist: List[str], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None - if not results.empty: - results['open_timestamp'] = results['open_date'].view(int64) // 1e6 - results['close_timestamp'] = results['close_date'].view(int64) // 1e6 backtest_days = (max_date - min_date).days or 1 strat_stats = { diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 4157bd899..977140ebb 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -85,7 +85,7 @@ def test_load_backtest_data_new_format(testdatadir): filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert set(bt_data.columns) == set(BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) + assert set(bt_data.columns) == set(BT_DATA_COLUMNS) assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -110,7 +110,7 @@ def test_load_backtest_data_multi(testdatadir): bt_data = load_backtest_data(filename, strategy=strategy) assert isinstance(bt_data, DataFrame) assert set(bt_data.columns) == set( - BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) + BT_DATA_COLUMNS) assert len(bt_data) == 179 # Test loading from string (must yield same result) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index f169e0a35..6912184aa 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -795,10 +795,27 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'is_open': [False, False], 'enter_tag': [None, None], "is_short": [False, False], + 'open_timestamp': [1517251200000, 1517283000000], + 'close_timestamp': [1517265300000, 1517285400000], + 'orders': [ + [ + {'amount': 0.00957442, 'safe_price': 0.104445, 'ft_order_side': 'buy', + 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True}, + {'amount': 0.00957442, 'safe_price': 0.10496853383458644, 'ft_order_side': 'sell', + 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False} + ], [ + {'amount': 0.0097064, 'safe_price': 0.10302485, 'ft_order_side': 'buy', + 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True}, + {'amount': 0.0097064, 'safe_price': 0.10354126528822055, 'ft_order_side': 'sell', + 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False} + ] + ] }) pd.testing.assert_frame_equal(results, expected) + assert 'orders' in results.columns data_pair = processed[pair] for _, t in results.iterrows(): + assert len(t['orders']) == 2 ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate assert ln is not None diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 94505e3ce..fca9c01b2 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -70,9 +70,14 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'is_open': [False, False], 'enter_tag': [None, None], 'is_short': [False, False], + 'open_timestamp': [1517251200000, 1517283000000], + 'close_timestamp': [1517265300000, 1517285400000], }) - pd.testing.assert_frame_equal(results, expected) + pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected) data_pair = processed[pair] + assert len(results.iloc[0]['orders']) == 6 + assert len(results.iloc[1]['orders']) == 2 + for _, t in results.iterrows(): ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate From 057be50941c25fb493b90086dabc7997987b7f05 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 11:11:47 +0200 Subject: [PATCH 045/162] Remove old concurrency method --- .github/workflows/ci.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3ed6d80d..bbe0bcf6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -300,18 +300,6 @@ jobs: details: Freqtrade doc test failed! webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} - # cleanup-prior-runs: - # permissions: - # actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it - # contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch - # runs-on: ubuntu-20.04 - # steps: - # - name: Cleanup previous runs on this branch - # uses: rokroskar/workflow-run-cleanup-action@v0.3.3 - # if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" - # env: - # GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - # Notify only once - when CI completes (and after deploy) in case it's successfull notify-complete: needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] From 9534d6cca177de8aee7edd330fc8103e8d07e4bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Jun 2022 07:03:40 +0200 Subject: [PATCH 046/162] Cancel orders which can no longer be found after several days --- freqtrade/freqtradebot.py | 11 ++++++++++- tests/test_freqtradebot.py | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 95eb911cf..d96c63bcc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -4,7 +4,7 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() import copy import logging import traceback -from datetime import datetime, time, timezone +from datetime import datetime, time, timedelta, timezone from math import isclose from threading import Lock from typing import Any, Dict, List, Optional, Tuple @@ -302,6 +302,15 @@ class FreqtradeBot(LoggingMixin): self.update_trade_state(order.trade, order.order_id, fo, stoploss_order=(order.ft_order_side == 'stoploss')) + except InvalidOrderException as e: + logger.warning(f"Error updating Order {order.order_id} due to {e}.") + if order.order_date_utc - timedelta(days=5) < datetime.now(timezone.utc): + logger.warning( + "Order is older than 5 days. Assuming order was fully cancelled.") + fo = order.to_ccxt_object() + fo['status'] = 'canceled' + self.handle_timedout_order(fo, order.trade) + except ExchangeError as e: logger.warning(f"Error updating Order {order.order_id} due to {e}") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 0e4f9db99..cd7459cbe 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4802,10 +4802,19 @@ def test_startup_update_open_orders(mocker, default_conf_usdt, fee, caplog, is_s assert len(Order.get_open_orders()) == 2 caplog.clear() - mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=InvalidOrderException) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=ExchangeError) freqtrade.startup_update_open_orders() assert log_has_re(r"Error updating Order .*", caplog) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=InvalidOrderException) + hto_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_timedout_order') + # Orders which are no longer found after X days should be assumed as canceled. + freqtrade.startup_update_open_orders() + assert log_has_re(r"Order is older than \d days.*", caplog) + assert hto_mock.call_count == 2 + assert hto_mock.call_args_list[0][0][0]['status'] == 'canceled' + assert hto_mock.call_args_list[1][0][0]['status'] == 'canceled' + @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("is_short", [False, True]) From 381d64833d30ee10684e0633826a473d4f873197 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Jun 2022 21:05:31 +0200 Subject: [PATCH 047/162] version-bump ccxt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index acaecd872..05d5a10db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.84.97 +ccxt==1.85.57 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From ac40ae89b9d50fba5bb088ba902e1ac6bce0f6a1 Mon Sep 17 00:00:00 2001 From: gautier pialat Date: Wed, 8 Jun 2022 00:20:33 +0200 Subject: [PATCH 048/162] give extra info on rate origin for confirm_trade_* Documentation : Take into consideration the market buy/sell rates use case for the confirm_trade_entry and confirm_trade_exit callback function --- docs/strategy-callbacks.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 656f206a4..b897453e7 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -550,7 +550,7 @@ class AwesomeStrategy(IStrategy): :param pair: Pair that's about to be bought/shorted. :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in target (base) currency that's going to be traded. - :param rate: Rate that's going to be used when using limit orders + :param rate: Rate that's going to be used when using limit orders or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param current_time: datetime object, containing the current datetime :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -599,7 +599,7 @@ class AwesomeStrategy(IStrategy): :param trade: trade object. :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in base currency. - :param rate: Rate that's going to be used when using limit orders + :param rate: Rate that's going to be used when using limit orders or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param exit_reason: Exit reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', From 7eacb847b05c53f7db80016885303be654cfb64b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Jun 2022 20:21:45 +0200 Subject: [PATCH 049/162] Fix backtesting bug when order is not replaced --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8fe5f509e..1aad8520a 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -966,6 +966,7 @@ class Backtesting: return False else: del trade.orders[trade.orders.index(order)] + trade.open_order_id = None self.canceled_entry_orders += 1 # place new order if result was not None From d265b8adb621f93cee91d9fdea85a52f9d425171 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jun 2022 03:01:48 +0000 Subject: [PATCH 050/162] Bump python from 3.10.4-slim-bullseye to 3.10.5-slim-bullseye Bumps python from 3.10.4-slim-bullseye to 3.10.5-slim-bullseye. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5f7b52265..5138ecec9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.4-slim-bullseye as base +FROM python:3.10.5-slim-bullseye as base # Setup env ENV LANG C.UTF-8 From c550cd8b0d2b8559f22a91c87e01c7afd1b00dd2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 07:04:46 +0200 Subject: [PATCH 051/162] Simplify query in freqtradebot --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d96c63bcc..fdccc2f8a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -227,7 +227,7 @@ class FreqtradeBot(LoggingMixin): Notify the user when the bot is stopped (not reloaded) and there are still open trades active. """ - open_trades = Trade.get_trades([Trade.is_open.is_(True)]).all() + open_trades = Trade.get_open_trades() if len(open_trades) != 0 and self.state != State.RELOAD_CONFIG: msg = { From 3cb15a2a5470e8a915aa5f39123808882b4b93eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Jun 2022 07:08:01 +0200 Subject: [PATCH 052/162] Combine weekly and daily profit methods --- freqtrade/rpc/rpc.py | 67 ++++++++++----------------------------- freqtrade/rpc/telegram.py | 5 +-- 2 files changed, 20 insertions(+), 52 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a98e3f96d..571438059 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -285,23 +285,33 @@ class RPC: def _rpc_daily_profit( self, timescale: int, - stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: - today = datetime.now(timezone.utc).date() - profit_days: Dict[date, Dict] = {} + stake_currency: str, fiat_display_currency: str, + timeunit: str = 'days') -> Dict[str, Any]: + """ + :param timeunit: Valid entries are 'days', 'weeks', 'months' + """ + start_date = datetime.now(timezone.utc).date() + if timeunit == 'weeks': + # weekly + start_date = start_date - timedelta(days=start_date.weekday()) # Monday + if timeunit == 'months': + start_date = start_date.replace(day=1) + + profit_units: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): raise RPCException('timescale must be an integer greater than 0') for day in range(0, timescale): - profitday = today - timedelta(days=day) + profitday = start_date - timedelta(**{timeunit: day}) trades = Trade.get_trades(trade_filter=[ Trade.is_open.is_(False), Trade.close_date >= profitday, - Trade.close_date < (profitday + timedelta(days=1)) + Trade.close_date < (profitday + timedelta(**{timeunit: 1})) ]).order_by(Trade.close_date).all() curdayprofit = sum( trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) - profit_days[profitday] = { + profit_units[profitday] = { 'amount': curdayprofit, 'trades': len(trades) } @@ -317,50 +327,7 @@ class RPC: ) if self._fiat_converter else 0, 'trade_count': value["trades"], } - for key, value in profit_days.items() - ] - return { - 'stake_currency': stake_currency, - 'fiat_display_currency': fiat_display_currency, - 'data': data - } - - def _rpc_weekly_profit( - self, timescale: int, - stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: - today = datetime.now(timezone.utc).date() - first_iso_day_of_week = today - timedelta(days=today.weekday()) # Monday - profit_weeks: Dict[date, Dict] = {} - - if not (isinstance(timescale, int) and timescale > 0): - raise RPCException('timescale must be an integer greater than 0') - - for week in range(0, timescale): - profitweek = first_iso_day_of_week - timedelta(weeks=week) - trades = Trade.get_trades(trade_filter=[ - Trade.is_open.is_(False), - Trade.close_date >= profitweek, - Trade.close_date < (profitweek + timedelta(weeks=1)) - ]).order_by(Trade.close_date).all() - curweekprofit = sum( - trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) - profit_weeks[profitweek] = { - 'amount': curweekprofit, - 'trades': len(trades) - } - - data = [ - { - 'date': key, - 'abs_profit': value["amount"], - 'fiat_value': self._fiat_converter.convert_amount( - value['amount'], - stake_currency, - fiat_display_currency - ) if self._fiat_converter else 0, - 'trade_count': value["trades"], - } - for key, value in profit_weeks.items() + for key, value in profit_units.items() ] return { 'stake_currency': stake_currency, diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e456b1eef..cfbd3949f 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -618,10 +618,11 @@ class Telegram(RPCHandler): except (TypeError, ValueError, IndexError): timescale = 8 try: - stats = self._rpc._rpc_weekly_profit( + stats = self._rpc._rpc_daily_profit( timescale, stake_cur, - fiat_disp_cur + fiat_disp_cur, + 'weeks' ) stats_tab = tabulate( [[week['date'], From d4dd026310b411ee78d7857dde4bec974226bb60 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Jun 2022 19:52:05 +0200 Subject: [PATCH 053/162] Consolidate monthly stats to common method --- freqtrade/rpc/api_server/api_v1.py | 4 +-- freqtrade/rpc/rpc.py | 55 +++++------------------------- freqtrade/rpc/telegram.py | 12 ++++--- 3 files changed, 18 insertions(+), 53 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index a8b9873d7..271e3de1b 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -86,8 +86,8 @@ def stats(rpc: RPC = Depends(get_rpc)): @router.get('/daily', response_model=Daily, tags=['info']) def daily(timescale: int = 7, rpc: RPC = Depends(get_rpc), config=Depends(get_config)): - return rpc._rpc_daily_profit(timescale, config['stake_currency'], - config.get('fiat_display_currency', '')) + return rpc._rpc_timeunit_profit(timescale, config['stake_currency'], + config.get('fiat_display_currency', '')) @router.get('/status', response_model=List[OpenTradeSchema], tags=['info']) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 571438059..a6290bd5a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -283,7 +283,7 @@ class RPC: columns.append('# Entries') return trades_list, columns, fiat_profit_sum - def _rpc_daily_profit( + def _rpc_timeunit_profit( self, timescale: int, stake_currency: str, fiat_display_currency: str, timeunit: str = 'days') -> Dict[str, Any]: @@ -297,17 +297,22 @@ class RPC: if timeunit == 'months': start_date = start_date.replace(day=1) + def time_offset(step: int): + if timeunit == 'months': + return relativedelta(months=step) + return timedelta(**{timeunit: step}) + profit_units: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): raise RPCException('timescale must be an integer greater than 0') for day in range(0, timescale): - profitday = start_date - timedelta(**{timeunit: day}) + profitday = start_date - time_offset(day) trades = Trade.get_trades(trade_filter=[ Trade.is_open.is_(False), Trade.close_date >= profitday, - Trade.close_date < (profitday + timedelta(**{timeunit: 1})) + Trade.close_date < (profitday + time_offset(1)) ]).order_by(Trade.close_date).all() curdayprofit = sum( trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) @@ -318,7 +323,7 @@ class RPC: data = [ { - 'date': key, + 'date': f"{key.year}-{key.month:02d}" if timeunit == 'months' else key, 'abs_profit': value["amount"], 'fiat_value': self._fiat_converter.convert_amount( value['amount'], @@ -335,48 +340,6 @@ class RPC: 'data': data } - def _rpc_monthly_profit( - self, timescale: int, - stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: - first_day_of_month = datetime.now(timezone.utc).date().replace(day=1) - profit_months: Dict[date, Dict] = {} - - if not (isinstance(timescale, int) and timescale > 0): - raise RPCException('timescale must be an integer greater than 0') - - for month in range(0, timescale): - profitmonth = first_day_of_month - relativedelta(months=month) - trades = Trade.get_trades(trade_filter=[ - Trade.is_open.is_(False), - Trade.close_date >= profitmonth, - Trade.close_date < (profitmonth + relativedelta(months=1)) - ]).order_by(Trade.close_date).all() - curmonthprofit = sum( - trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) - profit_months[profitmonth] = { - 'amount': curmonthprofit, - 'trades': len(trades) - } - - data = [ - { - 'date': f"{key.year}-{key.month:02d}", - 'abs_profit': value["amount"], - 'fiat_value': self._fiat_converter.convert_amount( - value['amount'], - stake_currency, - fiat_display_currency - ) if self._fiat_converter else 0, - 'trade_count': value["trades"], - } - for key, value in profit_months.items() - ] - return { - 'stake_currency': stake_currency, - 'fiat_display_currency': fiat_display_currency, - 'data': data - } - def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict: """ Returns the X last trades """ order_by = Trade.id if order_by_id else Trade.close_date.desc() diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index cfbd3949f..5efdcdbed 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -579,10 +579,11 @@ class Telegram(RPCHandler): except (TypeError, ValueError, IndexError): timescale = 7 try: - stats = self._rpc._rpc_daily_profit( + stats = self._rpc._rpc_timeunit_profit( timescale, stake_cur, - fiat_disp_cur + fiat_disp_cur, + 'days' ) stats_tab = tabulate( [[day['date'], @@ -618,7 +619,7 @@ class Telegram(RPCHandler): except (TypeError, ValueError, IndexError): timescale = 8 try: - stats = self._rpc._rpc_daily_profit( + stats = self._rpc._rpc_timeunit_profit( timescale, stake_cur, fiat_disp_cur, @@ -659,10 +660,11 @@ class Telegram(RPCHandler): except (TypeError, ValueError, IndexError): timescale = 6 try: - stats = self._rpc._rpc_monthly_profit( + stats = self._rpc._rpc_timeunit_profit( timescale, stake_cur, - fiat_disp_cur + fiat_disp_cur, + 'months' ) stats_tab = tabulate( [[month['date'], From a547001601f785f5c6d2171edc8a52159241e07d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Jun 2022 20:09:51 +0200 Subject: [PATCH 054/162] Reduce Telegram "unit" stats --- freqtrade/rpc/telegram.py | 158 ++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 93 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 5efdcdbed..e64ab7b8a 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -6,6 +6,7 @@ This module manage Telegram communication import json import logging import re +from dataclasses import dataclass from datetime import date, datetime, timedelta from functools import partial from html import escape @@ -37,6 +38,15 @@ logger.debug('Included module rpc.telegram ...') MAX_TELEGRAM_MESSAGE_LENGTH = 4096 +@dataclass +class TimeunitMappings: + header: str + message: str + message2: str + callback: str + default: int + + def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: """ Decorator to check if the message comes from the correct chat_id @@ -563,6 +573,58 @@ class Telegram(RPCHandler): except RPCException as e: self._send_msg(str(e)) + @authorized_only + def _timeunit_stats(self, update: Update, context: CallbackContext, unit: str) -> None: + """ + Handler for /daily + Returns a daily profit (in BTC) over the last n days. + :param bot: telegram bot + :param update: message update + :return: None + """ + + vals = { + 'days': TimeunitMappings('Day', 'Daily', 'days', 'update_daily', 7), + 'weeks': TimeunitMappings('Monday', 'Weekly', 'weeks (starting from Monday)', + 'update_weekly', 8), + 'months': TimeunitMappings('Month', 'Monthly', 'months', 'update_monthly', 6), + } + val = vals[unit] + + stake_cur = self._config['stake_currency'] + fiat_disp_cur = self._config.get('fiat_display_currency', '') + try: + timescale = int(context.args[0]) if context.args else val.default + except (TypeError, ValueError, IndexError): + timescale = val.default + try: + stats = self._rpc._rpc_timeunit_profit( + timescale, + stake_cur, + fiat_disp_cur, + unit + ) + stats_tab = tabulate( + [[day['date'], + f"{round_coin_value(day['abs_profit'], stats['stake_currency'])}", + f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}", + f"{day['trade_count']} trades"] for day in stats['data']], + headers=[ + val.header, + f'Profit {stake_cur}', + f'Profit {fiat_disp_cur}', + 'Trades', + ], + tablefmt='simple') + message = ( + f'{val.message} Profit over the last {timescale} {val.message2}:\n' + f'
{stats_tab}
' + ) + self._send_msg(message, parse_mode=ParseMode.HTML, reload_able=True, + callback_path=val.callback, query=update.callback_query) + except RPCException as e: + self._send_msg(str(e)) + @authorized_only def _daily(self, update: Update, context: CallbackContext) -> None: """ @@ -572,36 +634,7 @@ class Telegram(RPCHandler): :param update: message update :return: None """ - stake_cur = self._config['stake_currency'] - fiat_disp_cur = self._config.get('fiat_display_currency', '') - try: - timescale = int(context.args[0]) if context.args else 7 - except (TypeError, ValueError, IndexError): - timescale = 7 - try: - stats = self._rpc._rpc_timeunit_profit( - timescale, - stake_cur, - fiat_disp_cur, - 'days' - ) - stats_tab = tabulate( - [[day['date'], - f"{round_coin_value(day['abs_profit'], stats['stake_currency'])}", - f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}", - f"{day['trade_count']} trades"] for day in stats['data']], - headers=[ - 'Day', - f'Profit {stake_cur}', - f'Profit {fiat_disp_cur}', - 'Trades', - ], - tablefmt='simple') - message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' - self._send_msg(message, parse_mode=ParseMode.HTML, reload_able=True, - callback_path="update_daily", query=update.callback_query) - except RPCException as e: - self._send_msg(str(e)) + self._timeunit_stats(update, context, 'days') @authorized_only def _weekly(self, update: Update, context: CallbackContext) -> None: @@ -612,37 +645,7 @@ class Telegram(RPCHandler): :param update: message update :return: None """ - stake_cur = self._config['stake_currency'] - fiat_disp_cur = self._config.get('fiat_display_currency', '') - try: - timescale = int(context.args[0]) if context.args else 8 - except (TypeError, ValueError, IndexError): - timescale = 8 - try: - stats = self._rpc._rpc_timeunit_profit( - timescale, - stake_cur, - fiat_disp_cur, - 'weeks' - ) - stats_tab = tabulate( - [[week['date'], - f"{round_coin_value(week['abs_profit'], stats['stake_currency'])}", - f"{week['fiat_value']:.3f} {stats['fiat_display_currency']}", - f"{week['trade_count']} trades"] for week in stats['data']], - headers=[ - 'Monday', - f'Profit {stake_cur}', - f'Profit {fiat_disp_cur}', - 'Trades', - ], - tablefmt='simple') - message = f'Weekly Profit over the last {timescale} weeks ' \ - f'(starting from Monday):\n
{stats_tab}
' - self._send_msg(message, parse_mode=ParseMode.HTML, reload_able=True, - callback_path="update_weekly", query=update.callback_query) - except RPCException as e: - self._send_msg(str(e)) + self._timeunit_stats(update, context, 'weeks') @authorized_only def _monthly(self, update: Update, context: CallbackContext) -> None: @@ -653,38 +656,7 @@ class Telegram(RPCHandler): :param update: message update :return: None """ - stake_cur = self._config['stake_currency'] - fiat_disp_cur = self._config.get('fiat_display_currency', '') - try: - timescale = int(context.args[0]) if context.args else 6 - except (TypeError, ValueError, IndexError): - timescale = 6 - try: - stats = self._rpc._rpc_timeunit_profit( - timescale, - stake_cur, - fiat_disp_cur, - 'months' - ) - stats_tab = tabulate( - [[month['date'], - f"{round_coin_value(month['abs_profit'], stats['stake_currency'])}", - f"{month['fiat_value']:.3f} {stats['fiat_display_currency']}", - f"{month['trade_count']} trades"] for month in stats['data']], - headers=[ - 'Month', - f'Profit {stake_cur}', - f'Profit {fiat_disp_cur}', - 'Trades', - ], - tablefmt='simple') - message = f'Monthly Profit over the last {timescale} months' \ - f':\n
{stats_tab}
' - self._send_msg(message, parse_mode=ParseMode.HTML, reload_able=True, - callback_path="update_monthly", query=update.callback_query) - except RPCException as e: - self._send_msg(str(e)) - + self._timeunit_stats(update, context, 'months') @authorized_only def _profit(self, update: Update, context: CallbackContext) -> None: """ From b211a5156f5b7e92a652369ed1f6be19d3535b69 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 19:36:15 +0200 Subject: [PATCH 055/162] Add test for strategy_wrapper lazy loading --- tests/strategy/test_interface.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index b7b73bdcf..dca87e724 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -20,7 +20,8 @@ from freqtrade.strategy.hyper import detect_parameters from freqtrade.strategy.parameters import (BaseParameter, BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper -from tests.conftest import CURRENT_TEST_STRATEGY, TRADE_SIDES, log_has, log_has_re +from tests.conftest import (CURRENT_TEST_STRATEGY, TRADE_SIDES, create_mock_trades, log_has, + log_has_re) from .strats.strategy_test_v3 import StrategyTestV3 @@ -812,6 +813,28 @@ def test_strategy_safe_wrapper(value): assert ret == value +@pytest.mark.usefixtures("init_persistence") +def test_strategy_safe_wrapper_trade_copy(fee): + create_mock_trades(fee) + + def working_method(trade): + assert len(trade.orders) > 0 + assert trade.orders + trade.orders = [] + assert len(trade.orders) == 0 + return trade + + trade = Trade.get_open_trades()[0] + # Don't assert anything before strategy_wrapper. + # This ensures that relationship loading works correctly. + ret = strategy_safe_wrapper(working_method, message='DeadBeef')(trade=trade) + assert isinstance(ret, Trade) + assert id(trade) != id(ret) + # Did not modify the original order + assert len(trade.orders) > 0 + assert len(ret.orders) == 0 + + def test_hyperopt_parameters(): from skopt.space import Categorical, Integer, Real with pytest.raises(OperationalException, match=r"Name is determined.*"): From 88f8cbe17278f21d459a323d66d85cbe6c03db48 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 06:45:22 +0200 Subject: [PATCH 056/162] Update tests to reflect new naming --- freqtrade/rpc/telegram.py | 1 + tests/rpc/test_rpc.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e64ab7b8a..27eb04b89 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -657,6 +657,7 @@ class Telegram(RPCHandler): :return: None """ self._timeunit_stats(update, context, 'months') + @authorized_only def _profit(self, update: Update, context: CallbackContext) -> None: """ diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 95645c8ba..e1f40bcd2 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -284,8 +284,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert isnan(fiat_profit_sum) -def test_rpc_daily_profit(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, markets, mocker) -> None: +def test__rpc_timeunit_profit(default_conf, update, ticker, fee, + limit_buy_order, limit_sell_order, markets, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -316,7 +316,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, # Try valid data update.message.text = '/daily 2' - days = rpc._rpc_daily_profit(7, stake_currency, fiat_display_currency) + days = rpc._rpc_timeunit_profit(7, stake_currency, fiat_display_currency) assert len(days['data']) == 7 assert days['stake_currency'] == default_conf['stake_currency'] assert days['fiat_display_currency'] == default_conf['fiat_display_currency'] @@ -332,7 +332,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, # Try invalid data with pytest.raises(RPCException, match=r'.*must be an integer greater than 0*'): - rpc._rpc_daily_profit(0, stake_currency, fiat_display_currency) + rpc._rpc_timeunit_profit(0, stake_currency, fiat_display_currency) @pytest.mark.parametrize('is_short', [True, False]) From 1ddd5f1901d08073dd7d8c9cc3b819c728a20350 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 19:41:08 +0200 Subject: [PATCH 057/162] Update docstring throughout the bot. --- docs/strategy-callbacks.md | 6 ++++-- freqtrade/strategy/interface.py | 2 ++ .../templates/subtemplates/strategy_methods_advanced.j2 | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index b897453e7..410641f44 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -550,7 +550,8 @@ class AwesomeStrategy(IStrategy): :param pair: Pair that's about to be bought/shorted. :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in target (base) currency that's going to be traded. - :param rate: Rate that's going to be used when using limit orders or current rate for market orders. + :param rate: Rate that's going to be used when using limit orders + or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param current_time: datetime object, containing the current datetime :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -599,7 +600,8 @@ class AwesomeStrategy(IStrategy): :param trade: trade object. :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in base currency. - :param rate: Rate that's going to be used when using limit orders or current rate for market orders. + :param rate: Rate that's going to be used when using limit orders + or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param exit_reason: Exit reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 3b3d326ff..d4ccfc5db 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -289,6 +289,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in target (base) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders + or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param current_time: datetime object, containing the current datetime :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -316,6 +317,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in base currency. :param rate: Rate that's going to be used when using limit orders + or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param exit_reason: Exit reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index acefd0363..815ca7cd3 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -161,6 +161,7 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in target (base) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders + or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param current_time: datetime object, containing the current datetime :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -188,6 +189,7 @@ def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in base currency. :param rate: Rate that's going to be used when using limit orders + or current rate for market orders. :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). :param exit_reason: Exit reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', From a9c7ad8a0fcbf00063beba6a2b59809b99a97218 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 19:51:21 +0200 Subject: [PATCH 058/162] Add warning about sqlite disabled foreign keys --- docs/sql_cheatsheet.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 49372b002..c9fcba557 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -100,6 +100,9 @@ DELETE FROM trades WHERE id = 31; !!! Warning This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause. +!!! Danger + Some systems (Ubuntu) disable foreign keys in their sqlite3 implementation. When using sqlite3 - please ensure that foreign keys are on by running `PRAGMA foreign_keys = ON` before the above query. + ## Use a different database system !!! Warning From 3c2ba99fc480d028f8c6c86db68cfa5813b2b0e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 19:57:56 +0200 Subject: [PATCH 059/162] Improve sql cheatsheet docs --- docs/sql_cheatsheet.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index c9fcba557..c42cb5575 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -89,29 +89,34 @@ WHERE id=31; If you'd still like to remove a trade from the database directly, you can use the below query. -```sql -DELETE FROM trades WHERE id = ; -``` +!!! Danger + Some systems (Ubuntu) disable foreign keys in their sqlite3 packaging. When using sqlite - please ensure that foreign keys are on by running `PRAGMA foreign_keys = ON` before the above query. ```sql +DELETE FROM trades WHERE id = ; + DELETE FROM trades WHERE id = 31; ``` !!! Warning This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause. -!!! Danger - Some systems (Ubuntu) disable foreign keys in their sqlite3 implementation. When using sqlite3 - please ensure that foreign keys are on by running `PRAGMA foreign_keys = ON` before the above query. - ## Use a different database system +Freqtrade is using SQLAlchemy, which supports multiple different database systems. As such, a multitude of database systems should be supported. +Freqtrade does not depend or install any additional database driver. Please refer to the [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) on installation instructions for the respective database systems. + +The following systems have been tested and are known to work with freqtrade: + +* sqlite (default) +* PostgreSQL) +* MariaDB + !!! Warning - By using one of the below database systems, you acknowledge that you know how to manage such a system. Freqtrade will not provide any support with setup or maintenance (or backups) of the below database systems. + By using one of the below database systems, you acknowledge that you know how to manage such a system. The freqtrade team will not provide any support with setup or maintenance (or backups) of the below database systems. ### PostgreSQL -Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems. - Installation: `pip install psycopg2-binary` From 8fb743b91d33d7187c32765a6c6f3c2c5d7fd2eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 20:13:26 +0200 Subject: [PATCH 060/162] improve variable wording --- freqtrade/rpc/telegram.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 27eb04b89..106a5f011 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -605,10 +605,10 @@ class Telegram(RPCHandler): unit ) stats_tab = tabulate( - [[day['date'], - f"{round_coin_value(day['abs_profit'], stats['stake_currency'])}", - f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}", - f"{day['trade_count']} trades"] for day in stats['data']], + [[period['date'], + f"{round_coin_value(period['abs_profit'], stats['stake_currency'])}", + f"{period['fiat_value']:.3f} {stats['fiat_display_currency']}", + f"{period['trade_count']} trades"] for period in stats['data']], headers=[ val.header, f'Profit {stake_cur}', From dce9fdd0e4717559862b85df0850d5a1608e62fd Mon Sep 17 00:00:00 2001 From: Italo <45588475+italodamato@users.noreply.github.com> Date: Thu, 9 Jun 2022 20:06:23 +0100 Subject: [PATCH 061/162] don't overwrite is_random this should fix issue #6746 --- freqtrade/optimize/hyperopt.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d1697709b..ac1b7b8ba 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -429,18 +429,19 @@ class Hyperopt: return new_list i = 0 asked_non_tried: List[List[Any]] = [] - is_random: List[bool] = [] + is_random_non_tried: List[bool] = [] while i < 5 and len(asked_non_tried) < n_points: if i < 3: self.opt.cache_ = {} asked = unique_list(self.opt.ask(n_points=n_points * 5)) is_random = [False for _ in range(len(asked))] else: - asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5)) + asked = unique_list(self.opt.space.rvs( + n_samples=n_points * 5, random_state=self.random_state + i)) is_random = [True for _ in range(len(asked))] - is_random += [rand for x, rand in zip(asked, is_random) - if x not in self.opt.Xi - and x not in asked_non_tried] + is_random_non_tried += [rand for x, rand in zip(asked, is_random) + if x not in self.opt.Xi + and x not in asked_non_tried] asked_non_tried += [x for x in asked if x not in self.opt.Xi and x not in asked_non_tried] @@ -449,7 +450,7 @@ class Hyperopt: if asked_non_tried: return ( asked_non_tried[:min(len(asked_non_tried), n_points)], - is_random[:min(len(asked_non_tried), n_points)] + is_random_non_tried[:min(len(asked_non_tried), n_points)] ) else: return self.opt.ask(n_points=n_points), [False for _ in range(n_points)] From ad3c01736e74f4986cba86f685c2999fd202883f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Jun 2022 07:26:53 +0200 Subject: [PATCH 062/162] time aggregate to only query for data necessary improves the query by not creating a full trade object. --- freqtrade/rpc/rpc.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a6290bd5a..64584382a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -309,16 +309,18 @@ class RPC: for day in range(0, timescale): profitday = start_date - time_offset(day) - trades = Trade.get_trades(trade_filter=[ + # Only query for necessary columns for performance reasons. + trades = Trade.query.session.query(Trade.close_profit_abs).filter( Trade.is_open.is_(False), Trade.close_date >= profitday, Trade.close_date < (profitday + time_offset(1)) - ]).order_by(Trade.close_date).all() + ).order_by(Trade.close_date).all() + curdayprofit = sum( trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) profit_units[profitday] = { 'amount': curdayprofit, - 'trades': len(trades) + 'trades': len(trades), } data = [ From 7142394121abc4d511f110d805dd848989eb9126 Mon Sep 17 00:00:00 2001 From: Italo <45588475+italodamato@users.noreply.github.com> Date: Fri, 10 Jun 2022 09:46:45 +0100 Subject: [PATCH 063/162] remove random_state condition otherwise the random sample always draws the same set of points --- freqtrade/optimize/hyperopt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ac1b7b8ba..cb0d788da 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -436,8 +436,7 @@ class Hyperopt: asked = unique_list(self.opt.ask(n_points=n_points * 5)) is_random = [False for _ in range(len(asked))] else: - asked = unique_list(self.opt.space.rvs( - n_samples=n_points * 5, random_state=self.random_state + i)) + asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5)) is_random = [True for _ in range(len(asked))] is_random_non_tried += [rand for x, rand in zip(asked, is_random) if x not in self.opt.Xi From 76f87377ba542a106476828cd04846e29c0cfb88 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Jun 2022 20:18:33 +0200 Subject: [PATCH 064/162] Reduce decimals on FIAT daily column --- freqtrade/rpc/telegram.py | 2 +- tests/rpc/test_rpc_telegram.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 106a5f011..61b73553f 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -607,7 +607,7 @@ class Telegram(RPCHandler): stats_tab = tabulate( [[period['date'], f"{round_coin_value(period['abs_profit'], stats['stake_currency'])}", - f"{period['fiat_value']:.3f} {stats['fiat_display_currency']}", + f"{period['fiat_value']:.2f} {stats['fiat_display_currency']}", f"{period['trade_count']} trades"] for period in stats['data']], headers=[ val.header, diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 2bc4fc5c3..5271c5a30 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -447,7 +447,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, assert 'Day ' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] @@ -459,7 +459,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, assert "Daily Profit over the last 7 days:" in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] @@ -482,7 +482,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, context.args = ["1"] telegram._daily(update=update, context=context) assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 2.798 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 2.80 USD') in msg_mock.call_args_list[0][0][0] assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] @@ -561,7 +561,7 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, first_iso_day_of_current_week = today - timedelta(days=today.weekday()) assert str(first_iso_day_of_current_week) in msg_mock.call_args_list[0][0][0] assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] @@ -574,7 +574,7 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, in msg_mock.call_args_list[0][0][0] assert 'Weekly' in msg_mock.call_args_list[0][0][0] assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] @@ -599,7 +599,7 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, context.args = ["1"] telegram._weekly(update=update, context=context) assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 2.798 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 2.80 USD') in msg_mock.call_args_list[0][0][0] assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] @@ -678,7 +678,7 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, current_month = f"{today.year}-{today.month:02} " assert current_month in msg_mock.call_args_list[0][0][0] assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] @@ -692,7 +692,7 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, assert 'Month ' in msg_mock.call_args_list[0][0][0] assert current_month in msg_mock.call_args_list[0][0][0] assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] @@ -717,7 +717,7 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, assert msg_mock.call_count == 1 assert 'Monthly Profit over the last 12 months:' in msg_mock.call_args_list[0][0][0] assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 2.798 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 2.80 USD') in msg_mock.call_args_list[0][0][0] assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] # The one-digit months should contain a zero, Eg: September 2021 = "2021-09" From 2c7c5f9a6e0815760d1bafed9a96e8804c15b7b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Jun 2022 20:34:17 +0200 Subject: [PATCH 065/162] Update mock_usdt trade method --- tests/conftest.py | 19 +++-- tests/conftest_trades_usdt.py | 151 +++++++++++++++++++-------------- tests/plugins/test_pairlist.py | 4 +- 3 files changed, 100 insertions(+), 74 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 02738b0e9..b4b98cbeb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -325,7 +325,7 @@ def create_mock_trades_with_leverage(fee, use_db: bool = True): Trade.query.session.flush() -def create_mock_trades_usdt(fee, use_db: bool = True): +def create_mock_trades_usdt(fee, is_short: Optional[bool] = False, use_db: bool = True): """ Create some fake trades ... """ @@ -335,26 +335,29 @@ def create_mock_trades_usdt(fee, use_db: bool = True): else: LocalTrade.add_bt_trade(trade) + is_short1 = is_short if is_short is not None else True + is_short2 = is_short if is_short is not None else False + # Simulate dry_run entries - trade = mock_trade_usdt_1(fee) + trade = mock_trade_usdt_1(fee, is_short1) add_trade(trade) - trade = mock_trade_usdt_2(fee) + trade = mock_trade_usdt_2(fee, is_short1) add_trade(trade) - trade = mock_trade_usdt_3(fee) + trade = mock_trade_usdt_3(fee, is_short1) add_trade(trade) - trade = mock_trade_usdt_4(fee) + trade = mock_trade_usdt_4(fee, is_short2) add_trade(trade) - trade = mock_trade_usdt_5(fee) + trade = mock_trade_usdt_5(fee, is_short2) add_trade(trade) - trade = mock_trade_usdt_6(fee) + trade = mock_trade_usdt_6(fee, is_short1) add_trade(trade) - trade = mock_trade_usdt_7(fee) + trade = mock_trade_usdt_7(fee, is_short1) add_trade(trade) if use_db: Trade.commit() diff --git a/tests/conftest_trades_usdt.py b/tests/conftest_trades_usdt.py index 59e7f0457..6f83bb8be 100644 --- a/tests/conftest_trades_usdt.py +++ b/tests/conftest_trades_usdt.py @@ -6,12 +6,24 @@ from freqtrade.persistence.models import Order, Trade MOCK_TRADE_COUNT = 6 -def mock_order_usdt_1(): +def entry_side(is_short: bool): + return "sell" if is_short else "buy" + + +def exit_side(is_short: bool): + return "buy" if is_short else "sell" + + +def direc(is_short: bool): + return "short" if is_short else "long" + + +def mock_order_usdt_1(is_short: bool): return { - 'id': '1234', + 'id': f'1234_{direc(is_short)}', 'symbol': 'ADA/USDT', 'status': 'closed', - 'side': 'buy', + 'side': entry_side(is_short), 'type': 'limit', 'price': 2.0, 'amount': 10.0, @@ -20,7 +32,7 @@ def mock_order_usdt_1(): } -def mock_trade_usdt_1(fee): +def mock_trade_usdt_1(fee, is_short: bool): trade = Trade( pair='ADA/USDT', stake_amount=20.0, @@ -32,21 +44,22 @@ def mock_trade_usdt_1(fee): open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), open_rate=2.0, exchange='binance', - open_order_id='dry_run_buy_12345', + open_order_id=f'1234_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, + is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_1(), 'ADA/USDT', 'buy') + o = Order.parse_from_ccxt_object(mock_order_usdt_1(is_short), 'ADA/USDT', entry_side(is_short)) trade.orders.append(o) return trade -def mock_order_usdt_2(): +def mock_order_usdt_2(is_short: bool): return { - 'id': '1235', + 'id': f'1235_{direc(is_short)}', 'symbol': 'ETC/USDT', 'status': 'closed', - 'side': 'buy', + 'side': entry_side(is_short), 'type': 'limit', 'price': 2.0, 'amount': 100.0, @@ -55,12 +68,12 @@ def mock_order_usdt_2(): } -def mock_order_usdt_2_sell(): +def mock_order_usdt_2_exit(is_short: bool): return { - 'id': '12366', + 'id': f'12366_{direc(is_short)}', 'symbol': 'ETC/USDT', 'status': 'closed', - 'side': 'sell', + 'side': exit_side(is_short), 'type': 'limit', 'price': 2.05, 'amount': 100.0, @@ -69,7 +82,7 @@ def mock_order_usdt_2_sell(): } -def mock_trade_usdt_2(fee): +def mock_trade_usdt_2(fee, is_short: bool): """ Closed trade... """ @@ -86,26 +99,28 @@ def mock_trade_usdt_2(fee): close_profit_abs=3.9875, exchange='binance', is_open=False, - open_order_id='dry_run_sell_12345', + open_order_id=f'12366_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, - exit_reason='sell_signal', + exit_reason='exit_signal', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), + is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_2(), 'ETC/USDT', 'buy') + o = Order.parse_from_ccxt_object(mock_order_usdt_2(is_short), 'ETC/USDT', entry_side(is_short)) trade.orders.append(o) - o = Order.parse_from_ccxt_object(mock_order_usdt_2_sell(), 'ETC/USDT', 'sell') + o = Order.parse_from_ccxt_object( + mock_order_usdt_2_exit(is_short), 'ETC/USDT', exit_side(is_short)) trade.orders.append(o) return trade -def mock_order_usdt_3(): +def mock_order_usdt_3(is_short: bool): return { - 'id': '41231a12a', + 'id': f'41231a12a_{direc(is_short)}', 'symbol': 'XRP/USDT', 'status': 'closed', - 'side': 'buy', + 'side': entry_side(is_short), 'type': 'limit', 'price': 1.0, 'amount': 30.0, @@ -114,12 +129,12 @@ def mock_order_usdt_3(): } -def mock_order_usdt_3_sell(): +def mock_order_usdt_3_exit(is_short: bool): return { - 'id': '41231a666a', + 'id': f'41231a666a_{direc(is_short)}', 'symbol': 'XRP/USDT', 'status': 'closed', - 'side': 'sell', + 'side': exit_side(is_short), 'type': 'stop_loss_limit', 'price': 1.1, 'average': 1.1, @@ -129,7 +144,7 @@ def mock_order_usdt_3_sell(): } -def mock_trade_usdt_3(fee): +def mock_trade_usdt_3(fee, is_short: bool): """ Closed trade """ @@ -151,20 +166,22 @@ def mock_trade_usdt_3(fee): exit_reason='roi', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), close_date=datetime.now(tz=timezone.utc), + is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_3(), 'XRP/USDT', 'buy') + o = Order.parse_from_ccxt_object(mock_order_usdt_3(is_short), 'XRP/USDT', entry_side(is_short)) trade.orders.append(o) - o = Order.parse_from_ccxt_object(mock_order_usdt_3_sell(), 'XRP/USDT', 'sell') + o = Order.parse_from_ccxt_object(mock_order_usdt_3_exit(is_short), + 'XRP/USDT', exit_side(is_short)) trade.orders.append(o) return trade -def mock_order_usdt_4(): +def mock_order_usdt_4(is_short: bool): return { - 'id': 'prod_buy_12345', + 'id': f'prod_buy_12345_{direc(is_short)}', 'symbol': 'ETC/USDT', 'status': 'open', - 'side': 'buy', + 'side': entry_side(is_short), 'type': 'limit', 'price': 2.0, 'amount': 10.0, @@ -173,7 +190,7 @@ def mock_order_usdt_4(): } -def mock_trade_usdt_4(fee): +def mock_trade_usdt_4(fee, is_short: bool): """ Simulate prod entry """ @@ -188,21 +205,22 @@ def mock_trade_usdt_4(fee): is_open=True, open_rate=2.0, exchange='binance', - open_order_id='prod_buy_12345', + open_order_id=f'prod_buy_12345_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, + is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_4(), 'ETC/USDT', 'buy') + o = Order.parse_from_ccxt_object(mock_order_usdt_4(is_short), 'ETC/USDT', entry_side(is_short)) trade.orders.append(o) return trade -def mock_order_usdt_5(): +def mock_order_usdt_5(is_short: bool): return { - 'id': 'prod_buy_3455', + 'id': f'prod_buy_3455_{direc(is_short)}', 'symbol': 'XRP/USDT', 'status': 'closed', - 'side': 'buy', + 'side': entry_side(is_short), 'type': 'limit', 'price': 2.0, 'amount': 10.0, @@ -211,12 +229,12 @@ def mock_order_usdt_5(): } -def mock_order_usdt_5_stoploss(): +def mock_order_usdt_5_stoploss(is_short: bool): return { - 'id': 'prod_stoploss_3455', + 'id': f'prod_stoploss_3455_{direc(is_short)}', 'symbol': 'XRP/USDT', 'status': 'open', - 'side': 'sell', + 'side': exit_side(is_short), 'type': 'stop_loss_limit', 'price': 2.0, 'amount': 10.0, @@ -225,7 +243,7 @@ def mock_order_usdt_5_stoploss(): } -def mock_trade_usdt_5(fee): +def mock_trade_usdt_5(fee, is_short: bool): """ Simulate prod entry with stoploss """ @@ -241,22 +259,23 @@ def mock_trade_usdt_5(fee): open_rate=2.0, exchange='binance', strategy='SampleStrategy', - stoploss_order_id='prod_stoploss_3455', + stoploss_order_id=f'prod_stoploss_3455_{direc(is_short)}', timeframe=5, + is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_5(), 'XRP/USDT', 'buy') + o = Order.parse_from_ccxt_object(mock_order_usdt_5(is_short), 'XRP/USDT', entry_side(is_short)) trade.orders.append(o) - o = Order.parse_from_ccxt_object(mock_order_usdt_5_stoploss(), 'XRP/USDT', 'stoploss') + o = Order.parse_from_ccxt_object(mock_order_usdt_5_stoploss(is_short), 'XRP/USDT', 'stoploss') trade.orders.append(o) return trade -def mock_order_usdt_6(): +def mock_order_usdt_6(is_short: bool): return { - 'id': 'prod_buy_6', + 'id': f'prod_entry_6_{direc(is_short)}', 'symbol': 'LTC/USDT', 'status': 'closed', - 'side': 'buy', + 'side': entry_side(is_short), 'type': 'limit', 'price': 10.0, 'amount': 2.0, @@ -265,12 +284,12 @@ def mock_order_usdt_6(): } -def mock_order_usdt_6_sell(): +def mock_order_usdt_6_exit(is_short: bool): return { - 'id': 'prod_sell_6', + 'id': f'prod_exit_6_{direc(is_short)}', 'symbol': 'LTC/USDT', 'status': 'open', - 'side': 'sell', + 'side': exit_side(is_short), 'type': 'limit', 'price': 12.0, 'amount': 2.0, @@ -279,7 +298,7 @@ def mock_order_usdt_6_sell(): } -def mock_trade_usdt_6(fee): +def mock_trade_usdt_6(fee, is_short: bool): """ Simulate prod entry with open sell order """ @@ -295,22 +314,24 @@ def mock_trade_usdt_6(fee): open_rate=10.0, exchange='binance', strategy='SampleStrategy', - open_order_id="prod_sell_6", + open_order_id=f'prod_exit_6_{direc(is_short)}', timeframe=5, + is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_6(), 'LTC/USDT', 'buy') + o = Order.parse_from_ccxt_object(mock_order_usdt_6(is_short), 'LTC/USDT', entry_side(is_short)) trade.orders.append(o) - o = Order.parse_from_ccxt_object(mock_order_usdt_6_sell(), 'LTC/USDT', 'sell') + o = Order.parse_from_ccxt_object(mock_order_usdt_6_exit(is_short), + 'LTC/USDT', exit_side(is_short)) trade.orders.append(o) return trade -def mock_order_usdt_7(): +def mock_order_usdt_7(is_short: bool): return { - 'id': 'prod_buy_7', + 'id': f'prod_entry_7_{direc(is_short)}', 'symbol': 'LTC/USDT', 'status': 'closed', - 'side': 'buy', + 'side': entry_side(is_short), 'type': 'limit', 'price': 10.0, 'amount': 2.0, @@ -319,12 +340,12 @@ def mock_order_usdt_7(): } -def mock_order_usdt_7_sell(): +def mock_order_usdt_7_exit(is_short: bool): return { - 'id': 'prod_sell_7', + 'id': f'prod_exit_7_{direc(is_short)}', 'symbol': 'LTC/USDT', 'status': 'closed', - 'side': 'sell', + 'side': exit_side(is_short), 'type': 'limit', 'price': 8.0, 'amount': 2.0, @@ -333,7 +354,7 @@ def mock_order_usdt_7_sell(): } -def mock_trade_usdt_7(fee): +def mock_trade_usdt_7(fee, is_short: bool): """ Simulate prod entry with open sell order """ @@ -342,8 +363,8 @@ def mock_trade_usdt_7(fee): stake_amount=20.0, amount=2.0, amount_requested=2.0, - open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), - close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=5), + open_date=datetime.now(tz=timezone.utc) - timedelta(days=2, minutes=20), + close_date=datetime.now(tz=timezone.utc) - timedelta(days=2, minutes=5), fee_open=fee.return_value, fee_close=fee.return_value, is_open=False, @@ -353,11 +374,13 @@ def mock_trade_usdt_7(fee): close_profit_abs=-4.0, exchange='binance', strategy='SampleStrategy', - open_order_id="prod_sell_6", + open_order_id=f'prod_exit_7_{direc(is_short)}', timeframe=5, + is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_7(), 'LTC/USDT', 'buy') + o = Order.parse_from_ccxt_object(mock_order_usdt_7(is_short), 'LTC/USDT', entry_side(is_short)) trade.orders.append(o) - o = Order.parse_from_ccxt_object(mock_order_usdt_7_sell(), 'LTC/USDT', 'sell') + o = Order.parse_from_ccxt_object(mock_order_usdt_7_exit(is_short), + 'LTC/USDT', exit_side(is_short)) trade.orders.append(o) return trade diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index c29e619b1..c56f405e2 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -762,8 +762,8 @@ def test_PerformanceFilter_keep_mid_order(mocker, default_conf_usdt, fee, caplog with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: create_mock_trades_usdt(fee) pm.refresh_pairlist() - assert pm.whitelist == ['XRP/USDT', 'ETC/USDT', 'ETH/USDT', - 'NEO/USDT', 'TKN/USDT', 'ADA/USDT', 'LTC/USDT'] + assert pm.whitelist == ['XRP/USDT', 'ETC/USDT', 'ETH/USDT', 'LTC/USDT', + 'NEO/USDT', 'TKN/USDT', 'ADA/USDT', ] # assert log_has_re(r'Removing pair .* since .* is below .*', caplog) # Move to "outside" of lookback window, so original sorting is restored. From ab6a306e074da244c3798670cf00760a3c3c44aa Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Jun 2022 20:52:05 +0200 Subject: [PATCH 066/162] Update daily test to USDT --- tests/rpc/test_rpc_telegram.py | 59 ++++++++++------------------------ 1 file changed, 17 insertions(+), 42 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 5271c5a30..3cafb2d7d 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -27,8 +27,9 @@ from freqtrade.persistence.models import Order from freqtrade.rpc import RPC from freqtrade.rpc.rpc import RPCException from freqtrade.rpc.telegram import Telegram, authorized_only -from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_patched_freqtradebot, - log_has, log_has_re, patch_exchange, patch_get_signal, patch_whitelist) +from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, create_mock_trades_usdt, + get_patched_freqtradebot, log_has, log_has_re, patch_exchange, + patch_get_signal, patch_whitelist) class DummyCls(Telegram): @@ -404,12 +405,10 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: assert msg_mock.call_count == 1 -def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: - default_conf['max_open_trades'] = 1 +def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', - return_value=15000.0 + return_value=1.1 ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -417,25 +416,10 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) - - patch_get_signal(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade - - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobjs) - - trade.close_date = datetime.utcnow() - trade.is_open = False + create_mock_trades_usdt(fee) # Try valid data # /daily 2 @@ -446,9 +430,9 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, assert "Daily Profit over the last 2 days:" in msg_mock.call_args_list[0][0][0] assert 'Day ' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] - assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] + assert str(' 13.83 USDT') in msg_mock.call_args_list[0][0][0] + assert str(' 15.21 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 2 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] # Reset msg_mock @@ -458,32 +442,23 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, assert msg_mock.call_count == 1 assert "Daily Profit over the last 7 days:" in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] - assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] + assert str((datetime.utcnow() - timedelta(days=5)).date()) in msg_mock.call_args_list[0][0][0] + assert str(' 13.83 USDT') in msg_mock.call_args_list[0][0][0] + assert str(' 15.21 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 2 trade') in msg_mock.call_args_list[0][0][0] assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() - freqtradebot.config['max_open_trades'] = 2 - # Add two other trades - n = freqtradebot.enter_positions() - assert n == 2 - - trades = Trade.query.all() - for trade in trades: - trade.update_trade(oobj) - trade.update_trade(oobjs) - trade.close_date = datetime.utcnow() - trade.is_open = False # /daily 1 context = MagicMock() context.args = ["1"] telegram._daily(update=update, context=context) - assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 2.80 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] + assert str(' 13.83 USDT') in msg_mock.call_args_list[0][0][0] + assert str(' 15.21 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 2 trade') in msg_mock.call_args_list[0][0][0] def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: From 1a5c3c587d4936b9e6978197b2e257a7040ba5bc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 08:38:30 +0200 Subject: [PATCH 067/162] Simplify weekly/monthly tests, convert to usdt --- tests/rpc/test_rpc.py | 3 +- tests/rpc/test_rpc_telegram.py | 178 +++++++++------------------------ 2 files changed, 47 insertions(+), 134 deletions(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index e1f40bcd2..da477edf4 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -284,7 +284,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert isnan(fiat_profit_sum) -def test__rpc_timeunit_profit(default_conf, update, ticker, fee, +def test__rpc_timeunit_profit(default_conf, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -315,7 +315,6 @@ def test__rpc_timeunit_profit(default_conf, update, ticker, fee, trade.is_open = False # Try valid data - update.message.text = '/daily 2' days = rpc._rpc_timeunit_profit(7, stake_currency, fiat_display_currency) assert len(days['data']) == 7 assert days['stake_currency'] == default_conf['stake_currency'] diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3cafb2d7d..404fdd2b0 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -430,10 +430,11 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert "Daily Profit over the last 2 days:" in msg_mock.call_args_list[0][0][0] assert 'Day ' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] - assert str(' 13.83 USDT') in msg_mock.call_args_list[0][0][0] - assert str(' 15.21 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 2 trade') in msg_mock.call_args_list[0][0][0] - assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2 trade' in msg_mock.call_args_list[0][0][0] + assert '13.83 USDT 15.21 USD 2 trades' in msg_mock.call_args_list[0][0][0] + assert ' 0 trade' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -443,11 +444,11 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert "Daily Profit over the last 7 days:" in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] assert str((datetime.utcnow() - timedelta(days=5)).date()) in msg_mock.call_args_list[0][0][0] - assert str(' 13.83 USDT') in msg_mock.call_args_list[0][0][0] - assert str(' 15.21 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 2 trade') in msg_mock.call_args_list[0][0][0] - assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] - assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2 trade' in msg_mock.call_args_list[0][0][0] + assert ' 1 trade' in msg_mock.call_args_list[0][0][0] + assert ' 0 trade' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -456,9 +457,9 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: context = MagicMock() context.args = ["1"] telegram._daily(update=update, context=context) - assert str(' 13.83 USDT') in msg_mock.call_args_list[0][0][0] - assert str(' 15.21 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 2 trade') in msg_mock.call_args_list[0][0][0] + assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2 trade' in msg_mock.call_args_list[0][0][0] def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: @@ -487,15 +488,14 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: context = MagicMock() context.args = ["today"] telegram._daily(update=update, context=context) - assert str('Daily Profit over the last 7 days:') in msg_mock.call_args_list[0][0][0] + assert 'Daily Profit over the last 7 days:' in msg_mock.call_args_list[0][0][0] -def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: - default_conf['max_open_trades'] = 1 +def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: + default_conf_usdt['max_open_trades'] = 1 mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', - return_value=15000.0 + return_value=1.1 ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -503,25 +503,9 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) - patch_get_signal(freqtradebot) - - # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade - - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobjs) - - trade.close_date = datetime.utcnow() - trade.is_open = False + create_mock_trades_usdt(fee) # Try valid data # /weekly 2 @@ -535,10 +519,10 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, today = datetime.utcnow().date() first_iso_day_of_current_week = today - timedelta(days=today.weekday()) assert str(first_iso_day_of_current_week) in msg_mock.call_args_list[0][0][0] - assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] - assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 3 trade' in msg_mock.call_args_list[0][0][0] + assert ' 0 trade' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -548,44 +532,10 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee, assert "Weekly Profit over the last 8 weeks (starting from Monday):" \ in msg_mock.call_args_list[0][0][0] assert 'Weekly' in msg_mock.call_args_list[0][0][0] - assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] - assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] - - # Reset msg_mock - msg_mock.reset_mock() - freqtradebot.config['max_open_trades'] = 2 - # Add two other trades - n = freqtradebot.enter_positions() - assert n == 2 - - trades = Trade.query.all() - for trade in trades: - trade.update_trade(oobj) - trade.update_trade(oobjs) - trade.close_date = datetime.utcnow() - trade.is_open = False - - # /weekly 1 - # By default, the 8 previous weeks are shown - # So the previous modified trade should be excluded from the stats - context = MagicMock() - context.args = ["1"] - telegram._weekly(update=update, context=context) - assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 2.80 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] - - -def test_weekly_wrong_input(default_conf, update, ticker, mocker) -> None: - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker - ) - - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) - patch_get_signal(freqtradebot) + assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 3 trade' in msg_mock.call_args_list[0][0][0] + assert ' 0 trade' in msg_mock.call_args_list[0][0][0] # Try invalid data msg_mock.reset_mock() @@ -604,16 +554,17 @@ def test_weekly_wrong_input(default_conf, update, ticker, mocker) -> None: context = MagicMock() context.args = ["this week"] telegram._weekly(update=update, context=context) - assert str('Weekly Profit over the last 8 weeks (starting from Monday):') \ + assert ( + 'Weekly Profit over the last 8 weeks (starting from Monday):' in msg_mock.call_args_list[0][0][0] + ) -def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: - default_conf['max_open_trades'] = 1 +def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: + default_conf_usdt['max_open_trades'] = 1 mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', - return_value=15000.0 + return_value=1.1 ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -621,25 +572,9 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) - patch_get_signal(freqtradebot) - - # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade - - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobjs) - - trade.close_date = datetime.utcnow() - trade.is_open = False + create_mock_trades_usdt(fee) # Try valid data # /monthly 2 @@ -652,10 +587,10 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, today = datetime.utcnow().date() current_month = f"{today.year}-{today.month:02} " assert current_month in msg_mock.call_args_list[0][0][0] - assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] - assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 3 trade' in msg_mock.call_args_list[0][0][0] + assert ' 0 trade' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -666,24 +601,13 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, assert 'Monthly Profit over the last 6 months:' in msg_mock.call_args_list[0][0][0] assert 'Month ' in msg_mock.call_args_list[0][0][0] assert current_month in msg_mock.call_args_list[0][0][0] - assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 0.93 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] - assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 3 trade' in msg_mock.call_args_list[0][0][0] + assert ' 0 trade' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() - freqtradebot.config['max_open_trades'] = 2 - # Add two other trades - n = freqtradebot.enter_positions() - assert n == 2 - - trades = Trade.query.all() - for trade in trades: - trade.update_trade(oobj) - trade.update_trade(oobjs) - trade.close_date = datetime.utcnow() - trade.is_open = False # /monthly 12 context = MagicMock() @@ -691,24 +615,14 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, telegram._monthly(update=update, context=context) assert msg_mock.call_count == 1 assert 'Monthly Profit over the last 12 months:' in msg_mock.call_args_list[0][0][0] - assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] - assert str(' 2.80 USD') in msg_mock.call_args_list[0][0][0] - assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] + assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 3 trade' in msg_mock.call_args_list[0][0][0] # The one-digit months should contain a zero, Eg: September 2021 = "2021-09" # Since we loaded the last 12 months, any month should appear assert str('-09') in msg_mock.call_args_list[0][0][0] - -def test_monthly_wrong_input(default_conf, update, ticker, mocker) -> None: - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker - ) - - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) - patch_get_signal(freqtradebot) - # Try invalid data msg_mock.reset_mock() freqtradebot.state = State.RUNNING From 0a801c022316eb5a944f7690cc191d90a3364939 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 08:58:36 +0200 Subject: [PATCH 068/162] Simplify daily RPC test --- tests/rpc/test_rpc.py | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index da477edf4..982ac65d7 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -15,7 +15,8 @@ from freqtrade.persistence.models import Order from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from tests.conftest import create_mock_trades, get_patched_freqtradebot, patch_get_signal +from tests.conftest import (create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, + patch_get_signal) # Functions for recurrent object patching @@ -284,7 +285,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert isnan(fiat_profit_sum) -def test__rpc_timeunit_profit(default_conf, ticker, fee, +def test__rpc_timeunit_profit(default_conf_usdt, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -294,38 +295,27 @@ def test__rpc_timeunit_profit(default_conf, ticker, fee, markets=PropertyMock(return_value=markets) ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - patch_get_signal(freqtradebot) - stake_currency = default_conf['stake_currency'] - fiat_display_currency = default_conf['fiat_display_currency'] + freqtradebot = get_patched_freqtradebot(mocker, default_conf_usdt) + create_mock_trades_usdt(fee) + + stake_currency = default_conf_usdt['stake_currency'] + fiat_display_currency = default_conf_usdt['fiat_display_currency'] rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() - # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade - - # Simulate buy & sell - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - trade.close_date = datetime.utcnow() - trade.is_open = False # Try valid data days = rpc._rpc_timeunit_profit(7, stake_currency, fiat_display_currency) assert len(days['data']) == 7 - assert days['stake_currency'] == default_conf['stake_currency'] - assert days['fiat_display_currency'] == default_conf['fiat_display_currency'] + assert days['stake_currency'] == default_conf_usdt['stake_currency'] + assert days['fiat_display_currency'] == default_conf_usdt['fiat_display_currency'] for day in days['data']: - # [datetime.date(2018, 1, 11), '0.00000000 BTC', '0.000 USD'] - assert (day['abs_profit'] == 0.0 or - day['abs_profit'] == 0.00006217) + # {'date': datetime.date(2022, 6, 11), 'abs_profit': 13.8299999, + # 'fiat_value': 0.0, 'trade_count': 2} + assert day['abs_profit'] in (0.0, pytest.approx(13.8299999), pytest.approx(-4.0)) + assert day['trade_count'] in (0, 1, 2) - assert (day['fiat_value'] == 0.0 or - day['fiat_value'] == 0.76748865) + assert day['fiat_value'] in (0.0, ) # ensure first day is current date assert str(days['data'][0]['date']) == str(datetime.utcnow().date()) From 76827b31a9c59d0d7344ff379f5ef7f0fc1a56f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 11:18:21 +0200 Subject: [PATCH 069/162] Add relative profit to daily/weekly commands --- freqtrade/rpc/rpc.py | 11 +++++++++-- freqtrade/rpc/telegram.py | 12 +++++++----- tests/rpc/test_rpc.py | 4 +++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 64584382a..da5144dab 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -302,11 +302,12 @@ class RPC: return relativedelta(months=step) return timedelta(**{timeunit: step}) - profit_units: Dict[date, Dict] = {} - if not (isinstance(timescale, int) and timescale > 0): raise RPCException('timescale must be an integer greater than 0') + profit_units: Dict[date, Dict] = {} + daily_stake = self._freqtrade.wallets.get_total_stake_amount() + for day in range(0, timescale): profitday = start_date - time_offset(day) # Only query for necessary columns for performance reasons. @@ -318,8 +319,12 @@ class RPC: curdayprofit = sum( trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) + # Calculate this periods starting balance + daily_stake = daily_stake - curdayprofit profit_units[profitday] = { 'amount': curdayprofit, + 'daily_stake': daily_stake, + 'rel_profit': round(curdayprofit / daily_stake, 8) if daily_stake > 0 else 0, 'trades': len(trades), } @@ -327,6 +332,8 @@ class RPC: { 'date': f"{key.year}-{key.month:02d}" if timeunit == 'months' else key, 'abs_profit': value["amount"], + 'starting_balance': value["daily_stake"], + 'rel_profit': value["rel_profit"], 'fiat_value': self._fiat_converter.convert_amount( value['amount'], stake_currency, diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 61b73553f..c3e4c1152 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -605,14 +605,16 @@ class Telegram(RPCHandler): unit ) stats_tab = tabulate( - [[period['date'], + [[f"{period['date']} ({period['trade_count']})", f"{round_coin_value(period['abs_profit'], stats['stake_currency'])}", f"{period['fiat_value']:.2f} {stats['fiat_display_currency']}", - f"{period['trade_count']} trades"] for period in stats['data']], + f"{period['rel_profit']:.2%}", + ] for period in stats['data']], headers=[ - val.header, - f'Profit {stake_cur}', - f'Profit {fiat_disp_cur}', + f"{val.header} (trades)", + f'Prof {stake_cur}', + f'Prof {fiat_disp_cur}', + 'Profit %', 'Trades', ], tablefmt='simple') diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 982ac65d7..0273b8237 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -311,10 +311,12 @@ def test__rpc_timeunit_profit(default_conf_usdt, ticker, fee, assert days['fiat_display_currency'] == default_conf_usdt['fiat_display_currency'] for day in days['data']: # {'date': datetime.date(2022, 6, 11), 'abs_profit': 13.8299999, + # 'starting_balance': 1055.37, 'rel_profit': 0.0131044, # 'fiat_value': 0.0, 'trade_count': 2} assert day['abs_profit'] in (0.0, pytest.approx(13.8299999), pytest.approx(-4.0)) + assert day['rel_profit'] in (0.0, pytest.approx(0.01310441), pytest.approx(-0.00377583)) assert day['trade_count'] in (0, 1, 2) - + assert day['starting_balance'] in (pytest.approx(1059.37), pytest.approx(1055.37)) assert day['fiat_value'] in (0.0, ) # ensure first day is current date assert str(days['data'][0]['date']) == str(datetime.utcnow().date()) From 9ba11f7bcc0481bbc6db6c3faf3a9e25b8a0edd3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 11:26:49 +0200 Subject: [PATCH 070/162] Update docs and tests for new daily command --- docs/telegram-usage.md | 30 +++++++++++++++--------------- freqtrade/rpc/telegram.py | 6 +++--- tests/rpc/test_rpc_telegram.py | 32 ++++++++++++++++---------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 27f5f91b6..6e21d3689 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -328,11 +328,11 @@ Per default `/daily` will return the 7 last days. The example below if for `/dai > **Daily Profit over the last 3 days:** ``` -Day Profit BTC Profit USD ----------- -------------- ------------ -2018-01-03 0.00224175 BTC 29,142 USD -2018-01-02 0.00033131 BTC 4,307 USD -2018-01-01 0.00269130 BTC 34.986 USD +Day (count) USDT USD Profit % +-------------- ------------ ---------- ---------- +2022-06-11 (1) -0.746 USDT -0.75 USD -0.08% +2022-06-10 (0) 0 USDT 0.00 USD 0.00% +2022-06-09 (5) 20 USDT 20.10 USD 5.00% ``` ### /weekly @@ -342,11 +342,11 @@ from Monday. The example below if for `/weekly 3`: > **Weekly Profit over the last 3 weeks (starting from Monday):** ``` -Monday Profit BTC Profit USD ----------- -------------- ------------ -2018-01-03 0.00224175 BTC 29,142 USD -2017-12-27 0.00033131 BTC 4,307 USD -2017-12-20 0.00269130 BTC 34.986 USD +Monday (count) Profit BTC Profit USD Profit % +------------- -------------- ------------ ---------- +2018-01-03 (5) 0.00224175 BTC 29,142 USD 4.98% +2017-12-27 (1) 0.00033131 BTC 4,307 USD 0.00% +2017-12-20 (4) 0.00269130 BTC 34.986 USD 5.12% ``` ### /monthly @@ -356,11 +356,11 @@ if for `/monthly 3`: > **Monthly Profit over the last 3 months:** ``` -Month Profit BTC Profit USD ----------- -------------- ------------ -2018-01 0.00224175 BTC 29,142 USD -2017-12 0.00033131 BTC 4,307 USD -2017-11 0.00269130 BTC 34.986 USD +Month (count) Profit BTC Profit USD Profit % +------------- -------------- ------------ ---------- +2018-01 (20) 0.00224175 BTC 29,142 USD 4.98% +2017-12 (5) 0.00033131 BTC 4,307 USD 0.00% +2017-11 (10) 0.00269130 BTC 34.986 USD 5.10% ``` ### /whitelist diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c3e4c1152..2e1d23621 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -611,9 +611,9 @@ class Telegram(RPCHandler): f"{period['rel_profit']:.2%}", ] for period in stats['data']], headers=[ - f"{val.header} (trades)", - f'Prof {stake_cur}', - f'Prof {fiat_disp_cur}', + f"{val.header} (count)", + f'{stake_cur}', + f'{fiat_disp_cur}', 'Profit %', 'Trades', ], diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 404fdd2b0..11a783f3a 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -432,9 +432,9 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] - assert ' 2 trade' in msg_mock.call_args_list[0][0][0] - assert '13.83 USDT 15.21 USD 2 trades' in msg_mock.call_args_list[0][0][0] - assert ' 0 trade' in msg_mock.call_args_list[0][0][0] + assert '(2)' in msg_mock.call_args_list[0][0][0] + assert '(2) 13.83 USDT 15.21 USD 1.31%' in msg_mock.call_args_list[0][0][0] + assert '(0)' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -446,9 +446,9 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert str((datetime.utcnow() - timedelta(days=5)).date()) in msg_mock.call_args_list[0][0][0] assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] - assert ' 2 trade' in msg_mock.call_args_list[0][0][0] - assert ' 1 trade' in msg_mock.call_args_list[0][0][0] - assert ' 0 trade' in msg_mock.call_args_list[0][0][0] + assert '(2)' in msg_mock.call_args_list[0][0][0] + assert '(1)' in msg_mock.call_args_list[0][0][0] + assert '(0)' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -459,7 +459,7 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: telegram._daily(update=update, context=context) assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] - assert ' 2 trade' in msg_mock.call_args_list[0][0][0] + assert '(2)' in msg_mock.call_args_list[0][0][0] def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: @@ -521,8 +521,8 @@ def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert str(first_iso_day_of_current_week) in msg_mock.call_args_list[0][0][0] assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] - assert ' 3 trade' in msg_mock.call_args_list[0][0][0] - assert ' 0 trade' in msg_mock.call_args_list[0][0][0] + assert '(3)' in msg_mock.call_args_list[0][0][0] + assert '(0)' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -534,8 +534,8 @@ def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert 'Weekly' in msg_mock.call_args_list[0][0][0] assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] - assert ' 3 trade' in msg_mock.call_args_list[0][0][0] - assert ' 0 trade' in msg_mock.call_args_list[0][0][0] + assert '(3)' in msg_mock.call_args_list[0][0][0] + assert '(0)' in msg_mock.call_args_list[0][0][0] # Try invalid data msg_mock.reset_mock() @@ -589,8 +589,8 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert current_month in msg_mock.call_args_list[0][0][0] assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] - assert ' 3 trade' in msg_mock.call_args_list[0][0][0] - assert ' 0 trade' in msg_mock.call_args_list[0][0][0] + assert '(3)' in msg_mock.call_args_list[0][0][0] + assert '(0)' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -603,8 +603,8 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert current_month in msg_mock.call_args_list[0][0][0] assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] - assert ' 3 trade' in msg_mock.call_args_list[0][0][0] - assert ' 0 trade' in msg_mock.call_args_list[0][0][0] + assert '(3)' in msg_mock.call_args_list[0][0][0] + assert '(0)' in msg_mock.call_args_list[0][0][0] # Reset msg_mock msg_mock.reset_mock() @@ -617,7 +617,7 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert 'Monthly Profit over the last 12 months:' in msg_mock.call_args_list[0][0][0] assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] - assert ' 3 trade' in msg_mock.call_args_list[0][0][0] + assert '(3)' in msg_mock.call_args_list[0][0][0] # The one-digit months should contain a zero, Eg: September 2021 = "2021-09" # Since we loaded the last 12 months, any month should appear From 3a06337601b1ff4ca0609010635fb95b7eee7aa7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 11:28:45 +0200 Subject: [PATCH 071/162] Update API to provide new values. --- freqtrade/rpc/api_server/api_schemas.py | 2 ++ freqtrade/rpc/api_server/api_v1.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index a31c74c2e..11fdc0121 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -120,6 +120,8 @@ class Stats(BaseModel): class DailyRecord(BaseModel): date: date abs_profit: float + rel_profit: float + starting_balance: float fiat_value: float trade_count: int diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 271e3de1b..225fe66b9 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -36,7 +36,8 @@ logger = logging.getLogger(__name__) # versions 2.xx -> futures/short branch # 2.14: Add entry/exit orders to trade response # 2.15: Add backtest history endpoints -API_VERSION = 2.15 +# 2.16: Additional daily metrics +API_VERSION = 2.16 # Public API, requires no auth. router_public = APIRouter() From f816c15e1eb1452b332aa39bbd5d59b105a5324e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 12:02:41 +0200 Subject: [PATCH 072/162] Update discord message format --- freqtrade/rpc/discord.py | 91 ++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 60 deletions(-) diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index 43a8e9a05..41185a090 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -1,61 +1,44 @@ -import json import logging -from typing import Dict, Any - -import requests +from typing import Any, Dict +from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.enums import RPCMessageType -from freqtrade.rpc import RPCHandler, RPC +from freqtrade.rpc import RPC +from freqtrade.rpc.webhook import Webhook -class Discord(RPCHandler): +logger = logging.getLogger(__name__) + + +class Discord(Webhook): def __init__(self, rpc: 'RPC', config: Dict[str, Any]): - super().__init__(rpc, config) - self.logger = logging.getLogger(__name__) + # super().__init__(rpc, config) + self.rpc = rpc + self.config = config self.strategy = config.get('strategy', '') self.timeframe = config.get('timeframe', '') - self.config = config - def send_msg(self, msg: Dict[str, str]) -> None: - self._send_msg(msg) + self._url = self.config['discord']['webhook_url'] + self._format = 'json' + self._retries = 1 + self._retry_delay = 0.1 - def _send_msg(self, msg): + def cleanup(self) -> None: """ - msg = { - 'type': (RPCMessageType.EXIT_FILL if fill - else RPCMessageType.EXIT), - 'trade_id': trade.id, - 'exchange': trade.exchange.capitalize(), - 'pair': trade.pair, - 'leverage': trade.leverage, - 'direction': 'Short' if trade.is_short else 'Long', - 'gain': gain, - 'limit': profit_rate, - 'order_type': order_type, - 'amount': trade.amount, - 'open_rate': trade.open_rate, - 'close_rate': trade.close_rate, - 'current_rate': current_rate, - 'profit_amount': profit_trade, - 'profit_ratio': profit_ratio, - 'buy_tag': trade.enter_tag, - 'enter_tag': trade.enter_tag, - 'sell_reason': trade.exit_reason, # Deprecated - 'exit_reason': trade.exit_reason, - 'open_date': trade.open_date, - 'close_date': trade.close_date or datetime.utcnow(), - 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency', None), - } + Cleanup pending module resources. + This will do nothing for webhooks, they will simply not be called anymore """ - self.logger.info(f"Sending discord message: {msg}") + pass + + def send_msg(self, msg) -> None: + logger.info(f"Sending discord message: {msg}") # TODO: handle other message types if msg['type'] == RPCMessageType.EXIT_FILL: profit_ratio = msg.get('profit_ratio') - open_date = msg.get('open_date').strftime('%Y-%m-%d %H:%M:%S') + open_date = msg.get('open_date').strftime(DATETIME_PRINT_FORMAT) close_date = msg.get('close_date').strftime( - '%Y-%m-%d %H:%M:%S') if msg.get('close_date') else '' + DATETIME_PRINT_FORMAT) if msg.get('close_date') else '' embeds = [{ 'title': '{} Trade: {}'.format( @@ -63,7 +46,7 @@ class Discord(RPCHandler): msg.get('pair')), 'color': (0x00FF00 if profit_ratio > 0 else 0xFF0000), 'fields': [ - {'name': 'Trade ID', 'value': msg.get('id'), 'inline': True}, + {'name': 'Trade ID', 'value': msg.get('trade_id'), 'inline': True}, {'name': 'Exchange', 'value': msg.get('exchange').capitalize(), 'inline': True}, {'name': 'Pair', 'value': msg.get('pair'), 'inline': True}, {'name': 'Direction', 'value': 'Short' if msg.get( @@ -75,11 +58,10 @@ class Discord(RPCHandler): {'name': 'Open date', 'value': open_date, 'inline': True}, {'name': 'Close date', 'value': close_date, 'inline': True}, {'name': 'Profit', 'value': msg.get('profit_amount'), 'inline': True}, - {'name': 'Profitability', 'value': '{:.2f}%'.format( - profit_ratio * 100), 'inline': True}, + {'name': 'Profitability', 'value': f'{profit_ratio:.2%}', 'inline': True}, {'name': 'Stake currency', 'value': msg.get('stake_currency'), 'inline': True}, - {'name': 'Fiat currency', 'value': msg.get( - 'fiat_display_currency'), 'inline': True}, + {'name': 'Fiat currency', 'value': msg.get('fiat_display_currency'), + 'inline': True}, {'name': 'Buy Tag', 'value': msg.get('enter_tag'), 'inline': True}, {'name': 'Sell Reason', 'value': msg.get('exit_reason'), 'inline': True}, {'name': 'Strategy', 'value': self.strategy, 'inline': True}, @@ -89,20 +71,9 @@ class Discord(RPCHandler): # convert all value in fields to string for discord for embed in embeds: - for field in embed['fields']: + for field in embed['fields']: # type: ignore field['value'] = str(field['value']) # Send the message to discord channel - payload = { - 'embeds': embeds, - } - headers = { - 'Content-Type': 'application/json', - } - try: - requests.post( - self.config['discord']['webhook_url'], - data=json.dumps(payload), - headers=headers) - except Exception as e: - self.logger.error(f"Failed to send discord message: {e}") + payload = {'embeds': embeds} + self._send_msg(payload) From fdfa94bcc31b5fc751873ccfa943dc962a24a030 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 17:30:56 +0200 Subject: [PATCH 073/162] make discord notifications fully configurable. --- docs/assets/discord_notification.png | Bin 0 -> 48861 bytes docs/webhook-config.md | 49 +++++++++++++++++++++++ freqtrade/constants.py | 41 +++++++++++++++++++ freqtrade/rpc/discord.py | 57 +++++++++------------------ 4 files changed, 108 insertions(+), 39 deletions(-) create mode 100644 docs/assets/discord_notification.png diff --git a/docs/assets/discord_notification.png b/docs/assets/discord_notification.png new file mode 100644 index 0000000000000000000000000000000000000000..05a7705d7f599de625de31782344f98d93ccb17b GIT binary patch literal 48861 zcmbrl1#I2i+vV#pGcz>}Q^QORLmOslnDK-;4MUrxX_(V6r(tGhW@cti=DhDWpGNo2 zXzpLSlC5J|hO_Ox?PvYga`;z8DHKFPL;wI#WTZbU0{}!Wc)o;(1h4s0_1OUYgWw>o zBrSs7;mC;PjW^(gDLL5BSp04<8%d>3a68Au+sDjJI_ zivE1>5Ibh#1nK|2(rlg@Kt6zUF8p4l;NjY0lgirQbzhk|W3;qP2_)?Z&2kZm{=_2< z7V`)f^EjlAuD09uQr4J;R zHh6y=_dCbn4I5CysQc>`qTGJ&`bOgjquQ-o1;z!K9C&halFROeoK^!1bnz_ zBnduf(G#)=7x=&hVeCV6=qf+jDGK;+?_cx8(R<#qr9;WXUd8`yL?|VAk2|#gWCyQu zN|JG5*ZckcIV8(Ef^lTT{?X-x@k#|rn+=sfW30OVNFD%C)ZaYj{?h~RxU39+wII-g z=S+&ZN08~pXp%LLt%IE$**%S2H|Q6})n}~ek5^Jh2*tW+wnnyBzn%K4aYV?nSY=Zw zC8-*e0p+t2)X)H$qxqIevYt0$dW0yH(Oi@79FDDT;;KJ#{56s1xVz@_dboeH;cB)! z%WBa;Fq{^GsZ0Q}g8pZVZ&-`Tr%lDI3{m!&{q42cf1&(03EMvM4a`T6cgVv=S1YCY z?1&tKvciSh(>7*tMrGpl3oMF?1_To{O4v|D%kHu+AdM;aVy1C?owBOCxvs}Mphb1& zB%Tl(P@7s3vV$vpGU}q%Jxx4D=x~lvlp3V7X3`;nCH(V=U`o4qdW?=bF^nBHf+lG= zHx$oXTS62-zfVgZnejt5l{{dWy{U*#+cfr0Gnl1B*z~((E*@8`V)B?308*&1`a3=! zgrfogR&*}`E#w;78BSvl(tM71lK%lFkajK^#UCaKF83pzcUZBSoZ$n&D618 z$$YlP+DtMnPh_{_HHz!?ZDvBPk%aa&`W$`mF7NVsf05`_C6N&PQdH$aWBJ2q%82po z`my2npWRM|SN)Y1S=CGtzJRt)_Eqv_(>>pZyz_QPWE9Z005kYRgbCZ#cdnWI!?8lbl^E?!& zk|1sYUj?>U22J23+}l@gj#ly_v}JnDDRhrt41O?|CUT}n?V+j8G7meMAn0=db1NUT zd=#SqXB}4_ZU!1#gz-AsK;+f}jKflVQnu&1(57mOiVa3Yp*0PjxttcyqL!-&)P0X& z`|vwwAnT&bjiz@5cj7Nff*OhyrstUi&*Qn`GWLx|&Q>H8!M8wRH?^ZZ;TXtLe}zVa zT^!#84R?;EYqV57J5W8^X1_LT@S5B zx_aWTAV{)PoLc#wCp{Gn_1qHx@Nw39d^``sNT>F4ovwUJQOBD2=<&z{0bH27U(!u@ z7RmEY;GMgaasE;81Xx#XXqjS5Y_BnP!U>z!j>*1NOFLS9c2bNd1wIK~$kp71v;KIf z2^HgQxg`FHiWjWdDVji5&(l~AWzm)@dlz218O4HlZ5%iH%k<8kfWX-eAMBYm{iAQgWevZ!p@~Oq)i(7-)DS;rmAeDJ>)W2AyuzU+-ff*$P*tJR zTsdo*J|5@zER$i8j>tkN-gNz`z^uSR(hJEJ9-re*#l=YnW}S@Q`GVPxOeE^jHrmU8r@GXV%XH znXUA<+zMX<-8Xpl!hV2<$X4RIturi__C_ES>t{=4B{xE$l+XZFphN6$ofQE1QYirU zBAx~m=4uJl>?H>1V_m3q}9@8?C%8M2$={DQttbVfI1s%J91E6AkJqx$;`FJ6@ zm{zDt+V)KLMNPazT&Q-=iyHzxarh6cQC9PO8ldYRtHID)s1qT8m$Am9$FhXI72xvQA+dfPNLAYj z+pPOz^Ai9#mVAz43d;-Upw38Y7oz7Tb+rQ8++qaCGW1i+@!|(fAU6J?;B91vao=QT0FgR|lI{Xp&OQA2_ z?0a-1;bGlg*om_6i@&TS_{z(9X-h8wZQ(N^w6)z$+&tUu;tkfkt<%6z={9Iu-6IA` zkpUM27OSrV+&je;me|-JM4WL$SdjL78?*{059IL^$2$XTn6c`$o6ee|aQ|f7R11CS zcLFNWNBdp>=#$#uSCEONY2;jK9W+r=IZsgs$Mjv^+pb_0*^TgVgy8Soh$ zSgO>piO(pOvYBn^V4c?z5@^tJIhjGi+GCx+U@{VdMEinTIKHQ$)lE$Gpt;!Oi$;T) zK(ybJAhzy;7Xp|lFb^X_TLi(tH7W;;j;c3G1QA-+iOU62`+Fsnm-*ZpQvmhMT2FjRw#&(68 z`MKC0TT{Kk%NeSzfO{4e9wuYX_bD1Aw>g0x$Iyku>_``ffSWk5bb zSKL746*KCENlZg3O7Lxp5l_EqgvNwb#UUx&w=Kn5p0qD30Q+O6w;rHBbp50ueUv_dmmAou}sPSQ8M8z-beiA+2&(KcG zNGb`PG~zYaX!JmWvAo)Q!#Fkbl{aj&rMwFC*xGhUny*t%Hu+L%Fp?b?DI>}=O|y5s zvW2MCS}Nn&#!6j!9bcaNx{bE4CT&_RN0al>`YZY&AE`m5s1IE3J5U z_TNvOhIZ`1EO+i?dyQ?ChstZP_Qb>jTSz9=?2~r*{_z?+8EJ9!GAgzbgOh4PdfmF(!?ww9^y)(^cQLd4~r#Yx=CQK0{Ektz=YH;K}cub$cA ztSN1(%eL)@<>A2fMz|FCb>7jcajs8E{$AF2EV9LuU(PfnEjMln+s4Xg#*9VVGLW7s zUa-L(F@=_nz5Y;UO}|+@?;LqyD^nRPWg`N(c89i;C9Ui&+IH`x(0jA84FX(a?Sk?ByCEk?2Ulq zda0)4w#N^j&PvT*0$GKnOiT7+?>4fCc{1caXYioU=#Ml@f^SHROTm9gGuR z2!J&cpctn7vg%i?40SXQ<%EZ19xZ3@W7IZsyMM5=dn*j-FQc#lK(8>%4&IIHwQ@KT z7s?4Kme7czvzmXB)Lp}=&6kD)z$Hy301Pzw#FFh^MpQJ_;OSCrMDiRzTmI1s1e7vPiDd z9a(>sHFre3ZQtQ9^T<3offxLzrs97}=^hg$5jVZb$K<1oCpHXN5}@N;523;OA?B2! z#W7uXEy4(BwOad(i3^Z#x&g@kGljJuE>^wMB92y9`;-TW<`eRzmV=;_=ka8gZ`{Ct zjj==t^Iosp4p(B30DDiL=L_8wD7zH{7jF|nf^1x-kfNZB;bITM{}3;*5*XW@5Hae~ zq>|DcK05mwFT!QcC|CYh$nF~j!mCKAlV8KJSL~ZE`LXOUoG<#O#8x$HPk#V8Uy`#> z)If=X{p=wEQe_{4<^;|OY$uu)$#JB{$M1bOhkCzvA6nNQ@yA6tJTm`0=atDa-ohbe zGs-Cn6u@=7y^s=r-X{rXMym-Kh}@-<;=>Qv1}=*G(}L~nNAC9_X(2X3L-Lo6L~G_v zzqXt?w7xkNoNe(g%b##{pdkE?~nP- zX=(22#ySIQ(dqScldFUZ@(Sj0*T^{w)a6ZEKejnB2-u9OC``%rBIK}**xE`y-64D+ z9O(_B*1Nv+NNxOwTJ=L%P|?7PYXvqUK?8ym`NH05$L_i+$*$zh#iH4i@_^qrG@;T^ zZSkGpE0;5A7xvjb^Dwo)G89#jg1)>OGml{+Qlr^D+f%7Yh9M56XExe0=I)ftAN%uv z9ruNO+v4tJ#&A04s9$@RCOyUYD}v1-?te&#cweE3nfRUq(u45drAJ1-K>KGx_jG#a zXZ_eUtsI}MHtp@?mw|Y#OS~D%@M*h#PxV-9IuH|oh~6tuEWc>I44AUld0=Quiu`ZNPW?S@#oPqHr{3#p8$E?nCsfz0(+%+K^B+`sIihTpR9p$E2E z*|s7T)(knIfFfJEC(70vIQFlriRT730_cgWRyr@d?BETU#Gdr3uPFpUmvtX(FW%ti z69FK1y2vay7TVwI$a%TW_ceaM$+&9??Ou9hby}sMLhimSat%~Og8%>v0Yu*x4eSTG zMwVMMGFs2_a-QwCX6rg-h z$7^rYTxOv5u;c`QS?5s2&~YVNd$%iiEF^vYgdCT$OgEXf#MeKvR^k<%$`r`f#4 zzn?Z<{t-rXg6S57`e@0p?PZVG?sJF>pOO`iA{bBTBOse^)wt`uYO6Of0QQEfc0s})yhau@tK7O zqKYClzBl|Jl!UobJYCFPT(=51dl|ceU)SjL9;Y1_Oj^AV@QH~!B|CcZ(QjGeY9-6P z@-y?+rR*R`&TdBL31M4 zqLGd?Rv%(Y1Z^#hS>}5aw?9j z;YSs&U$HTRuEc%h**wPPzwek61UbK{=-!+z1ZkA&(IB!oeAIKwa-0Z)_HdbxGQ+EV}J+zb%Y5HjPU`hN(&qf2!}e6i}@-Z_@}zIptvAtCYG)(??@?_Szp*iJv%lfSFtwP)!UE<2#cLvaCG zhFo1jA&EJO1zP!{C37byH2*#xBnaz|u?MsHOsa}^qDq|4+G(ukJn_84?5uXXB9ms8 zGuK2f_syNpCcy>qece^28M`J@O7y7pL%qsF*=^NuHZJ-_ihqmihYtY8R)yoH~sXw)^48^q!%ZTdT?pHPKXj{v?aa5Mm9}F74hKW^qQRF zEJ9)>m$6mxmCd-V9wZ%a*8^3LQ)3fACXv%pJCkbqbNp2!7Sw#YCVPkQZ)v%hEB+EA z&rP(v%skJzMry2+724#m9^N}VPn!0QDU)*k!f;;*9aU>!;;2z7fG$GKWcXIZ{60*v z(!XGxok@ar^m3`Qx;ZOl{oXT0tE&WmE^GGNRtH5SInjmo!?($pAZu3E_>j5cT>k5_ z+&Gbaq(sIDhaNz>8|CoQXE!|^>!fyK{L_9iM8~(5RdtP&zY?Dog5UjszVe{5DwO0A}MSn*k-6!`UYR@xuZ6)8Yx-9%X;eUY=chf+ncL}_AyY?@bd z^i?J@0+B_XAz-`4K~M7Tu48fz8S7_$`MB&0U6lXAuhS|@-+79+0rtP_*!l-+-E>M~ z5E9f(42gA+2Kc|-Pp%z0r{P`e_>b8L@L1)*dhH73qN?reC7d9KNi3vqqCInLBCWIXu zn}B*x1ikRk`}R1uFxm>_cV=vbnzX&WBhG%j3Pz8PANZ6kY1#JEpqc}Qdvh5S|1UU- z=77RY<4MGcCOoP?Oulr=SQ%1UGjqXThg23=Vxy5x0huy+^@K#+-ARuk*Q4Ihu<#_klcx(op|M(??Q?lh{HC;<8r!nbOoA}@fqW* zBqv({z$&t9tiLR48s{^aoysnk_eo1_t*$XGW7i*~&db)d(Zjy)tkK+c+!-!)O`iw$ zLBjKR*oFL|Q0w@SSoIBGvg739^4B7LHIw>RlELUG-c`O7cE!1_(mB+%%)wpEELfGY zWN5K&LfFYoFS-dtNY$@zO&&2>`zN3I2%oM~7?? zz2CPA>Gg|$v=Flj0-hg4kXk7622<`H-?{7**PG15z5MyE^ zhE%kQw7zk6b8>tfxfbT)M{?GwqoMd-F}6yOIOjjEpNP4AN%Q3YckoRpS5!~WwN{^T z7C75HpSKR(YBS2Npl|10&gzB{J?3UHp>N~W$ptS*O6Yd12(84U@4?;{OexNxGk1o2 zX?U#(FD()CMs=bI?<8~Fw9dDArn~z5x5&i`JkZsU*N6iZ7WRGU^WL+&N4Us&9mr%S zpkh9c&{oNV0ucRSvP4LqvK_uR?e)-AA!O~Vf=873r+ofI;zVWRz#RS)`>o&{t*E1B zz%Z6f`_3^2_SzBgA}uo75Lh|OIDz>rd+$eVkOdc#q0=&G9syU6m^s~k#R9e-i)*SF z&^UB-tCG7M;P&8;nGXUm=JOu*{yPg0J;;{1Q|}7+3)oTjVYAY2mz0*At<^_=wLv`gtyE}`>3YHFemYF)FUn zDVV?A)?5ywaj3K;LFkLN|4Rx8{#~xvcwn$|hI0RV+`rA8U$31t)x{8n39GQ`cPnI& zRmbb(pr*LqjporOxG23(jz12;YWa&B;=!keF(Qr~oDuj(%KPI~7Km9uW*MQ1IB&g- zm&oJ7xXj};jdl4HoDTEBjVdI7`}^0mnNIAF@i1VAB)+JI!Tg+&{!a=tsd)iJ<0H2GSFO)mX`?&^Op*N}o1Yk&sfCT)!;O_wHIl@Y)CCW&^ z#9!2*E9O81b#f&OzwkrHa9S)QAS*4I01vRj0UHjM_l?M?(Dxa~!_fX0j56*TDH}|F zzGseqZbO4V0DyEQ@S+%p3m`6f?6iz*R(GhDC~1QaDvJUE`7sjqKHP<6EY`Rn!z9nx zlUau^9-%4H6em5ipVr=AbF$j0# z!h>TX3D|emG*mu(Rxmq<4{r##xJ!BT{hrF$dL7fV6{e*qq8iC?>oZ*aP_b!fHCyN# z2tmKTwN!uey6OB)9nJ~6C^B)s@M0oYjJtoW5g&|6n)pwFkQ&EJ3AfTgT(Rq)_m~a_ zeq^u*DMJkJw&|E;!qe-gpv@J>{%O8>ABT49vjhLWO|Z>q1ueph8>l_*mq)0n0GlHb)h5n0>{Ii=EB<)Lpvg;qbL|1)}+0>WIoDtx2|~8f6q# zn6(-UQ#seL{|zBH#)wL#0GXJ-(L?ckWBqE)QB85ygL|P%*8;9AA>Aw z3;$slGx70aIaE+QDb+g*Jy*{p)ptWVLyd56KcPrb9_NY*iT&8Tn6gi7yTDK<^JR9T{t$f!LDUG9U}oaSG{yJ_6&zt9{ub!hX3!-4$7g*$Ec3)AKn@hLWxG}cn2nOuUqy* z8R3)i#xzst*`QeWsPkZ=xm?eXuKKrZnM4W!l=18LivHbM5 zX5R(gLpY`=zM!pMuU3YiDi?jzojhZ0ANfPSo!t}h2KD$9}Ro1h@$x@D9Nko%x$A8fgdH{HcyuYf=&Mn8wl))U%luydN zSPRz~O3W{pRZcz}JMZZ$O>L*y)ICq^V>f$Ut=aC%ut=VhH~nHz>oF=ed&GiLm%4Jw zV$YNuuxceGIwq98R+9c{&h(7$2^8J5-vi2;IwyR7llfkEvym~f)wH~7YZt;!W*y53 z4z4jSX(ta^A}21Xv~d@gw9|MLX&OP`b5S^*LY}0i6HT)W)P5;^RRq01m##;krE7po z`-MAD2!5*IcR-d{G=f*|My&30A3tN!4C2=BV~T>8T&v&; zOZ7Whred!M!!W-~)0iQ|-3!yOK~da~8%NWLQ2vzN#iB)xXrRNF{oCWZ$AX~0>nsY= z{H~pcw`ck(MT7wUkpth#Ywq=eLs@hb3c8aH!;BfC*Kf4?+vU0d2I709lb1T5 zx8*vmFdz4>*ciUYpakEi+-1}{f})j<+sxAB0$H8{n5?D;#e`D@g|Ep)qzq_bEnw1b zXb7K+J2lK@Uq)S+8(42cs7Cq@gXsLj5Px;j_R{=0@tZ2^o2J{@u)yH>Yw>grUn{r0 z$IIP~e;pAn*9`HBm*vKNd6oGHu~(Tp83Yhz8Bb7O&MRe$KyoZykW#Dh=6Lm2b=7|E z2i|O~jD7psF-hld{`zHtaz^?>SnW@<9&f~F?PFXhzu^46+TOWpIjWWbfn~-XeSyU_ zhS^oOz^9c)E-D~t_K*27ached1kiD6k0h%$=NmG|v*O4D<3DP8jO_#ojMSGGNvGM7 zKJZA8hCN!XmA87$mFw5qc)Vo8-sL>pKi%Dy;>Sao#aIn5yX8F$-D}^Se2u*Zg8jRY z(qs~Uai&1{cbUF(uUh6QIl1>QEi}}^9W5Xk;E*Y(6OY^BSIS6b z64&|rUN)S1C1E^YnNX3WcQ%-_uADv<&DKqmkl3!JS{(A}&f3cUZe~eD6mh$C;(Bkk z?bycl`1)P@G6;k{T39+z(!g(2Jz5|D6q07kHB`5TG7tD!(7ig1_aO(J!y!rDdR0-+ zzMfAnm5h~?wvX&J9}O3<4BVOltR=mVoMswm`W0K^ohxg0E++S$j{K&1@wuI>9kdjy zx34IjzXDqqqoGKJXJoP?C1u&^Gc4wbJgzcq&Sx!Hl~)P6fl@e<)Kd!^d$JSLn!Y9$ zByVLQg%sc6i8t}iwHo?m$bn2r2*O^Qv$h=<#~r85{;RXM;!Dre-)yn3m@7v$Kk(b$ z{M1xUK)5x9_Afn2Tp6fSXVPN9UawSR#;&;%yAN|jQGrP5C2UhZTa|U}!naJ&s&9b{ z1Cvubd28cCH z1L3q5H>s)PoJAD?5W7JE?}g{4i0r_!;MW8BdR#&(WD~B0yBp}}TD%m=;Lq~!NH{>D zuF-|+c0K2-*=s{G(HjMUkbSUlA|l|DK`Sf$J7M^_oZf40h%O?UY(sGU~ zV?{&YflkyhFKBX7(B&1mE>~iio?%RgLjBGto*Ua$hJ>uu^S+^u8qBDRTOIzje+)4nu;)@sWYv`hdA&{Mn|u`vS-ra#?89KhW0?sI+qq8FQ%H_ zoX#bZ8Ho|6l3Znttg}XWe}F2A4JvrYO3}0Ts!cq{GW* z3t~}Tk-D;ygT*o?+Hc#~{t;70<+I7tZ06O>l==Y2b#z!*jcP}EOrc1&LZyt=K<{F z?_YxdD_7vV!9oZLa9Lj7{d#N~aqFF{&=!u8O1>L+4wqhP{Vd{3!(0k;##1k2M~wV7 zGiD0ilV3_`P`l|}J7kg?y6_`A$|cEKRTn=qpKcYCz#tzEFhs03rG^Ptmq<{fnG@QG z4HJkX>TTW{$E+(F7tG&Jg-nsiz-Xtq{sc zJzWBA41BIt$0{kFj(ZwTSPtIb?KnoPb&BMj+|}e|GxF3adEIE$7kK~nrPOikj(BLb za>had7oILgtsl%m;5#4&rYla&bgC2e&PU|%vIC4oaU8SidBH9|tK*$iPG8Jq67mis z{sl1Gy*Un)HBOZM5DtMqW@V>7;{#{7)pP}_Q>qMfgKO9WM5W`_G@3@q3b!7YMw6BX zY6C~qI8uq8e1>+8G*h^&0hD1cvg)xL5=7p3#5!9z8F3np98t@>?Y`4@72wGGh-qhj z+CTY-ye$)D@0EPx#p4&^CXJ&nPkwI5JC+ zE$rc4NUvWl;&%P0xU2F%e@7H`epJ=Rg2;+BBN=GSV?${}`Y3T4o*rT8OIbOlO=(+j zYOoLKKTn0ZEin?ZQy8}6sf$mzCCR}ME>^@4W@@{9tbALsH5P47w~sE%*V&JY0Y z%e`x9>fl5Xib(DTeo>)7LP83)I=im8ahJgRw642+X`TOJbIxBMiL#I2hGu+6Z{u?_ z*VTp+^3s!yf+4HPqg6$ zt2bRE2;y@-dkDZe^>`RcLFwB|K-$SoiopH_U+%XS|7EVm0^Z3=)Gd1zwAgjJ6T zkQ5KG7s$4=KSIN6;d-pSsU+b2C`c%>fZQx~6lAZ1;`8vZlmSm#43m1sc^uTAn5hMC zi5d}DiTgpfrg20cG{feqRd~P6`nW$ z8P&qlHZ`$aACs2J*~Vo)v3?K`l3S5=5)hv=HL<@kM)z?m2SMOZCbjIOUo_Y>%J>S{ z?T$s!_mZS5m%$l2HoX5SX8c(KAq^T0SCIgH*O{;(J@wQ&s=69lxe?%X2!2yx9T`;+ zp-LIYpCt#~-OI`vSkn9c+bLcnBIxx8M0Oj&D zuJfc1x~zBpc|L&z*(%MBKC6$3fE3qe4mxaG;4hio<^h{=tgAqX(EX0HWz@Utn zIl9GD1&#eNBi`|FoHQdhpisS#3HYoa!L-TUy=&)-i1&J_6(Oq~0wcZR^9 zH&p)o{w5({$X}lf31m$Rt}q`6p8A9AiWR>wIxUkVP(=dqZ?@qY#QXOdKw3e zz<05!!tnIIepqCE!3$ot5-8q?nciV%Rrdp>>{R1(U1VUxZ8hB$=w57i$l;$eLCU7+ zaQVb*zO)ZcMTkL!onK*1^*{pv1r9fQO{-7h<2A_*4xZBdZ8F#QSX5~bN!hoXUmG1H zyv)u-+6aVx5s66gs(T*`X5F|BI|dnE#KlfGFt9r~^n~@VNQ$O^&-yitnGafV>X&hB zzR-!;z4kkfa@Cwg+BB@lLnBIr!(HyJLkqPL1A@SW1F=7sQy}Xv>ZE27Lj`E0%i#*Z z+h|7;>;CvA;!~C4LAwJ8UX>OeZjFfjLD>7wanFql$vx(OMpZQaiK;NzR`bWUqeFjv z2w*$T+ObJ_$aWRYv}AxSb&g_x1EG2+z8}U?%H)cfw-yQ@f-P2KMQj~MwQOa#4f_Yg ztXxrTxrGy5nM_NEw3yD}2X}j=hj*M1#Ut4_h5tpSl>U#rNI9g#lp~)}M)A2g|BN4H za#ob0xEBPQomm05!d(2BsmXkKB4!$6c2H?@F!{6LuDi3xlJb~}j>;w` zE@SPtD@9^36iMxTRv*i9vb?fWu4%4PK@A18A1pqFHY5?JDwh*cMTDzonBc2 zIMfuy!|_gjhFN1zt-ZW(W@qDu63TnG;E?ZB6)r(A#E75-(o|5aJ2+M^QvY^zs6ypDI7yX= zP4)IC!b(dbiZV8wD#4zGQzsqKSpMy$zABWO(iQ3B2Dhy7aWyEq*Aas>{VY5aQhiYQfL?3LJA2y|1p!$N*4?tnohKX7qUffCE}<0fu+ z*;54dP8zlX^M2{T!P#)Uu%qol6Jtr420dtI#-r2kKA;-qId6^{ErgB9sv8pxt~dO_ zXKD>}sTzHW?5(CoPh{amHq;YF<+zgb#r5Vy0nvUi<9-Rt4f}O-{km%Zbd4-;6|eP+ z>pNmz^OILxVoyXf0GxQ}VA9BL$MvAyqk1&e(9%;*BnKFYyfKbS3wZRxvqo|I!0PF* zU1imHE?l8~s|;FEL@~|FZHhEAN<`~9f0~3xz8@%K$h9k9p<#y#zE@mPZsP!{#SO{3 z`OocvUlJe1L9b)6DiJZo?#bj^>q_q0aSc@wBtQc`=5 zUmp2wo`t)QM#GJi!0%GOjmu@`c8gu++apkb&wtTMUam~9ayRWj>Q)i+QJ}@d77hVNNmchZeylys#~`8c zS17Uj90;#rzzZH$A3f_q(5`?DH*GERylG#ZpN@L3KdVBHnrQpZ&n_PlqG6E|1Sbi= z&wUImDD!E1wZHhJ7qQ!+Oz-_qWFxM2v|bf^O(P^)ab(c|5YjgU3-!*)418UE7do!r z65m$?vy(cXbc#N+apym%>T6#;9mXw_=a2!jvb-D^qJ4pmwoaeV8AmUrR%HYo-+pEa zX+0mU?_U6b;llTUTBJ&QS3CK3@_AHzlD~i8?oDv#SIAlZy>kK`3*jwyzgsfI z_Y}8ln2RNu>eVTVTN;1KxS@kdj6oildSc) z_m8pa^?O9kG}10~hjkz0dulUf_H{?@ji^C1T;MUf}9~(N%m!b>(w)S{>*5= zYV;t;?omR&Jpr>zP6L~8KNo@a`k1u%QgWsHq%=4VsNV1L3nt)2W3BH@8yF$a^4ZL}Zzf6?IbX3o#)2keiDO?t(W1$({cV}^;iE+qKfPj0H5x=b1SD(B&o3Zs$bmk!Uv^T`0l#73p7*&6S$Ude`gapMlN3zyucuov5 z)BD{8iM%N_++TnC^x^h*GQDPlslwm6rCPU}Fil7r)P9ZiPbxGOVYkg*-pfalLBV60 zY!Hlqao`-I$uGOe_=BQA34ZnOf6D~Rs`b!;ujzPYSMRqybFEal6{ktU?g`|#BN0-8BfqpU-v_uhCp_CxbV(X15H2 zuV$52_s(xRU6PPG)U7l+TE{CZrj^w-mK`t^Gg(47CKl2es7*hDAjn$`FIpDTs2>DAh#U`ER!P0PC8Ixo+$o zSu}~is5i%YKAIfpK0Qmwt$Y+7FasSf@=39rE6p*kfODF)pTB`kIybdbxeQ;w(~KV1 zYdH2J1u}VcLt7bwiangyoiqM$MAm`3HMNAZNSy@Da*%#X!+!2q%3+BcwZBA3Low&J zO+e_rCUMA*gKIMre^IN{|EB@N1!eI^V~IxoodM`5v^+Brzb$KXt_G=Sn~(^*oy!UMQvb?JO*b}BcI>^HVeMlEhaT6{;DMa_A0uqlnLs|N$F&<9PhDw2G>gvhH@ z7InL22j%DX@cGy$Lv|csBw&5-Q(39OHEYubBb4<0-T?nxj(aCc46;O_1Ocb6c+HMqM&aCZ;x1ZdpdUFPI}&&)gTGjl&R zH8WMz{?MF;(_OuL@4c?IewU9L?P$fB&RVBmUHow34U@a^X@M%r@4C@AmO89-vs1P8 z(dT>7MJuL_k+=h6d?u&uPi`4JC>b4R#lPt-kzszptlUg&A4lCZhYP5OvHzA)dC_gC=ZmnUW(LhnjRj|13zdqzS_@9*QT<{(vrrbl5U) zIxLVmPNZXww@lHF5!ab%Pf@yZBn}PeI)t?h1Gl|(^Y0dFA#=%yE<16<{8=CI)zBv2^`UhV*B~Pv*apJTFk2ZVD4DLN;aOp^_HWl_KQ7CDc zbsDlb<{xLXu`F5qNs~pdqFLOh*_t}?QeG9|q*Ko9vA=~<)>mY;*^YL}Kd=4n3*jE6 zMLR3?%eB@*-FkZYUO0B#hK4l*5}iYOHYtPx?Hl*SROS{`;UipSbWYx&_{@S}6SHw| zErG-m#Z>RxZ-qG=GET>)+}1$oU7t*t*~Z@ZMIs#5F{)Rq7c6eT*JR5T1(A#z7yDPN z{Clf$ZBS;@lJ6hBYFoa%uV-9VXhe(9)8Ipf0e~OT(iNwC7>^H2%YH~(X>uEBMG5Mq z-()-yJ3jM`$E%_oy|lw*NWp!pc={+>=O;ITHttTj@?XneZe{Y#oi+#h`yDbJm-=kD z3s~WcF+HAj#y;>%Buuljh2ME0e;S%MW_WtvJ-7jGXIT{x3{rn~?e9*z^|TgsIhCyt zXux7+Z)sr}J>k)4`B=vDnXR7h}?AV(x#WLhAjV@YN`* zib}TgktO=AE|j2pSMi{Xlscnpluo}av)R%L$r?UgHWvLE56L5xiL{l8v9|qa5`#el zkN*$f$~zO(?ZHfFl&V<~&)3IuPKK81v3NdI$*(z<>ps7fp(?FlzM5(iOL7_-cPi7} z5wESCmd!GSkYR?IJrJnX4jEHjz%iHRs02DIN@hOq)@`)H>u)D}4{E1XcV&Em1GrQO zKYft+3V!Tnozu_u_6KPn6l9#o_O6YhaAs`3cYCH40hmqi9I|fcU%1?1fUr3J=}08R zjB9+b))xC;IUO)doWk7)$Vk3f7_D3$TYYxcK!Qu+->dA z@g@@6oe7lZ8rC^6te#X^fAXSqoNh$ocWQd4gS%DNld-jxs25XvXF7X@J1pr_Xmofu zpfCr_^6uG`RH5aMBw3^p*{n*YY;7$Sgn~<|A`4gOc9UAl_tA_q?fx(Kjw@jkcrd-0 zjnY}`!PqTQ6=kxEbOb*IlRcMq*G96yFRBzn6%PBI#z`oj{X_P-Ayqj?WS}o7pSk$= zH}{(R3cEjLh+fuH4^N=q?2Zp&Cgki`qJf39G41MAAqVBd$4Gg4)f75AIP1j#`7X7eK)oh@T`jE-ew zc;#mx?->-8_9 zi(~wU3oB`FyEc9#2rTeJb(f) zhF_)KTf64rW6*0+>f(z~9HQ1K*Vk-XOVZE6ge9#`$}b$K|5QA_jJnaNaFCw43T>#N z4sX*po19w{ShjBWdR{HBLWtBXLoq^NPUBar{32BAbBA;H?CIJ5Y+gDN9~Ocn!+ncr z_CB1J-%uM~7Uz&3qwO)2H?$K4+pmhIVz%xNXen6wSzTe?JDCfECh)!S$j&T&sqQPz z1=d~r3i?=7i$lo^@nN!3yP)BaO#h z6DA|Dq>ykpSlG(0OAl6eUaA#rkNe;d^kjP-sm5M?!|q5++s{9C%=zI_fpQ5?p@{$3 z^*bZNnCCDQxL1mBu7FfZ8{vA-n4$z$TF$l?es`v*drCa_H#o-gk3n;NRhGB+O~MkH zgkWy`sHAlu6rsL#-hKytOw;}Gmxjw_0^fEsIe>(VA7S`4T;M8bzEWu1&25gIa(8?e zOE60Bv@~4%f&0nt-79?^*?+N@{x8PBWQTlri^UJ12=?4^IVn*mEVC#Z8xgW+hNiwK z4?fDL?d4k7dESF^h1&X9HTdVQ#_|&gkFut3y__8S>1EAv&VtIi2Xt&*d6*=8_~$9o zm`Gb#R99OG8I8H_LDaS%UVY~&AkLQg@$S`m>SytP5K;eTbZ^LYA8|kjp50&Cj-bJR zU$EBx@>F3d=dE2yi;L^@@~l5H;#yw|oW}o1IMN3%pg74((Du4B`aM+9B8gYJ!Zy!D zK=N!igl`Y#{l-FGo^RTE&wuoKtT4^1+82t8HhX(p9njb1{r#=a%W}JQPT5%Lja6L@fAF>saWAKPWMe@%*XCV7%wI;=kN>CRjpj$9 zIey~@l^?Ugrgg~ac$js~6+ZpHXuZ~C8@z_rW9sS8y4+qO-pLGpSz_FIucv+H`s4F_ zT`!<3XH2U}Cbxji|JpKarxYMj-uj5rVc2YAleiLpGG)x{9#<4e8!|1D(BmrShNbal zhi&9ml4|MPbHO6NTzBNK42!5jY)xcyQ#kTR@g|;x%-dEH*i={BhjmYf{tw$HMjzvI zVVP=N*Rlq$4Eyyve|H@+q>JOfEZ?s?0fD*``-Pc%$2{=HOr}bXlc_N7Ar;V4k2AeO z8KW!|Qf_HgMcIcvl`UV(%y89f{dC2Tk8PZ^_SYSS@mJwRevA&k7lH{F{6#SNI!F%87n|QfS4UBKxjK&K@QCQr@pdI5~CNHw! z11;aH6LD3=K8q%T1ek6u23fxFvgJ%AJT4uT912>0?UFe?&{Ir79^o52j&TG4E4J!;$>ETm1r98%%b*6P%{pFb+p zsb*34aH)H<_sm>1R}sa6qEIra%EJa@5WdX>#ex|>C+VFf5pHpcooU0bS|`L2VM;l` zF_j6QI%MBxyJ~m(gYWhG0ULt0Tv7rwpkA@raJr{H2Z50`u)hlFw+Tm^5dqdp+lr2TJP@bPt8 z?sT(C<+B=O6PrH{6M=M{5ox(;s{MA2J+^7c6{B8g&d!C|{)fSO^F?7l7dVo=u)jns zRN*`&_{A;I)U;qXcwh6D7r21D^Nv4S_mxZRGWtX@I=EkAWVGUm#T<(YG*~YsNmNyR zSv$rBzWmfIE#yylI{>@zQ;Acw8fmIqHR9Q9=LW74iZcBS2Q#E~J>7B{^+JX_qc#Xx z;Ehm*{D9N#6k+cmMU@@spO+neKbBq)?VyY@wyPzc1&w_~&k_2)ol}ctHoM6qtENTy zhrK7ZW!SkrY^Dv4rA%=8U_tHd&y0$RsBurau3qnEviG|cLVxDmnet$*!HwA%Nh*lJ z^2b;KF<1#IE)3?o7@MV>+`s954wmxoL<*f-QUj}9ryoCS>UdE$n+HLt%W6vFfiRNj zJI~|mLIJ@OP9cpRS2I+QU1OiYGF>0uNdG`ZJBee>l|;w+=Ww^$RN`w5KuQ(4HT#yu z%jS}&e6C;4m(KsOBVFG!vi~WojH*bs7RIM7EBEqHrYDFPxmm47X@^P?7T8T0i;LI# z7{Zlu@-bg+Jaz^gQpW`^2g zBGW01(ZjbI8(|b^An*#cdCogJ2kvWl_w7=UbC1{XWJ}XyPY)8p=c39>k1{kC_~wnC zLgCeISfzT)cyyrK#$uc(i3FSXhh_0tRO&$zo?lg={TxWV3JFamwLw>1Cw^UbsoA$( z){k!?W9QKHJ8ke?U+UB;G(BRo%_$Z z$%;O%Dyb{?N$IOL4bDG_^%sdL_2`lLT(K>mtTS&6%!VaKw18B!1OD6Zp4{ujEBo^TZRvC zzkBt_U||*%#BOhNf3lD%WcI?j?GKWzoYw0*kCzv_P1^|2I$ld`LZJL3xAkQW2pboDKM9~w{e|ma04(VtVbu=aPUn9H0W|zW4xFbGLW+8rqgU<`qhs2`7Pih;{G{2 z=w0{m-qoHPLy{yp5J73qn@_a&{;gBGGpkbkPslYoK2F_DMw>cv?9Qa;=j^*_@w*m| zERIhJpUNho4=U0>L)mV23sbeZc(w#NUCnRVhmUy5`(Z@cbTwBiT#EV`uGO8tcZ!DZ zNG7Uv;mPMrvfV5md0WNvU=w%qwZgVDW=g+Hw+oje99p+9iZJFaz@jpdZPb~XmvK;| zo`$$_I?oY>`f<`&Vn;1uV*N6QZ}w!Wk`RhUs5{GY4+kNc?Z7W;J53IG2U*$BSgI*= zdSYI?jjwe7iWy%i>s!}Srd*Y$lu}3ZQg8OIjOS)BR%>!+c5@DeFKrOdPwEyI4yOW# zfr^lKV}&N!G6meR(XMh54KAQ8hK8dMJLY_>CM0~H&%4FHCr-h(tWBzxpf!V~&3>_A zAdh^2IE!h!+u|dRa`XoVFMypZ0nNX2+qRL2pJz6RKj|WZFIvLW#*c-%xxKNyq@Pc# z>61$F^lKkx+jljnJ81oZ-th&X@bqN&>BLajFTO*o!HNgllpxf(B6#rlj2gaKsYiV#H;u#?-{-T|n$0n`>z^#t1v1>rYg9bmG~}PYr^6V1)OFW>gf)9~ zKVYr9`Ftq#tBQd;OeKYDBi19kKTR(yZG6e1l!`OSu_k*ZrGu}Zus7xcv5UAqn^l}$ zDw39H(rAMrV+35P4LJyF7+@f;&L$cveh+C4!vR1{Ls>DGJ;#$z&5LCi0nMOhZ}aJ& z$1*JR$|KUqvD1%4OJ>EC&OK&8DI%Yhq`ZPw&OK2_cGC|So}^mO>il2o2R_x1{9D+v z^Q7%Y3a)F_3mE#Uhb1!=6)i<|>mttzM9Ac+_>>~fp!KB7@`v`*+{&L@7tZY0_PLe7 zquz%HRKd+)+`oIW2+I&&c)(@%xymJ9+H19Xv;0Ao|4sgO_Rq3B&?oBBC)TFENl?4i zEfH!rU3Zo49GG0%GhMtaC1KwaXtU+8$v7^?93!)<=b(B#7)c z?}{Ek)I#rre0ggZYIGG>#mtH5yF}bw*NJC-E&i3or?H*7>gA3PiT}P((w6Jf0Gv1R7@kN|UF_CQ|QR%SiPf0NC)0Mcg?&_GMz(p3IhTN)g-WE0Z08 z4@F2w`Uor%yZ*sWmcr?QFy2GN*NT_rJIaaQmoALe?@65u>VB)GHn1*P4pbVMn<_>F z7Hrl7=&n}3{Lq!C_P=3W6|65Y0-;Ph2X9foV7}QSX`&e}D`R;QfLmM7NNi$?LZ8dK zGod%L#Zm5eTB*xwyl zruKvvNi)w~Hi*7*6=ocDy1DcSE&JF4N2;aTagoGGOgdW7cl`5h2a7AbyA(+M!+oc( zhCk&@LOu80*rOvjARBLkDLG?<+n$&F$h=We9Pkl0qJ5W(rzStK-6(#B`8mCzc?l@J z(|;dpG2g0h7J&`;Knt8f0S!riH+sLxf;f|E2ckK@Lyx;~Tq}h2-lVRh;qx{${^G3Y za0CGPbnG`8LgPJ;=*#DwwuF5R?R(J0%oIc3@>8SJLee)An!}VW`)#fi2~Jn4_gUbY zMY_ZLf*bwvQ;XH^fk@oFj9a4}H0@cbkdaWn7i02tg$t)mT<CPNBU461Bmx4}hXFREEH}^K+nAD0z5z@y9rU$mWj6t%M!lFC3Za85u zjPS4Z#VRo%ED$DOHLO#BGuJbS&3CLDs8Q$U88a;$gy=KN6!5!Ov3$lHevdyyzYiji zy0W2ckBZ9;8sRz?>Mbdb69FH8egMQLUU3;OeyJ$WZwV{x)bra zSZxjEm6dqi_b;2+TS?Y(!@P1bpZ2&WmXShHZ~U`atKU+Koisq}&;f~Fq_2=yEx^za zc^Ipme{K1(Gy$UFm+h))yMs9?ko$aM&Ag9{b%w@&u0W3hXq^O+;n}wM!cHRER#z8-VDFPs`{b45fai;>wm+9l$ zfad|xaxMLw;gjJ-7{t_S{VKy9derM%6?bIt9$h|p2B8PPHyD$=vrS0GH%-(*2 z6)HOoIH#rFJUL$Eu{JWa^%qi?Z&iF3w&WZ{VfpO0!>XRt(3Evkt@<~#s&dNi*&F(- zg2^@LWNVFU!jmC9ZFt1vk>4a}#JeBG+AD84?qNI3AJhfUJ0kop_4B64QRT)&2`YXB z%(#g3>wv1;>YX#FrZ2BuL()g_g6Z;qKS~xQvwb2n&f2jq2;q_*t5-Rc;G<~|K3jPc zK9lw{buWlwUDI4ry~rwzgNyIIab=KwL6+-jnQ4v5IFIc(w}+&PLW6tn`n(4>VH30u z=^S+j|M>mJP(L0Yb@tO)Muhk%CGy99{O8plbXfK^cflD(KUZ?4l8LanVjoh}XU3e{ z0HDSslJJ|ULHmQ5Vw8lz+Fd6)3QIC+c;9K*d|Sn7s-*3xTY5Bj0C>q32oLSr*x2}B zaxqB06*>`lJnpftniaD;riFD!S5^vm%G|1*#mp_9%*Ue&oNWY@udJM(?pMOPI-#!M z9H_4U#5kBYVb70`rWrMTzHIFD-KbGR#t|Rq{IK~ci}SI!u5e1>c|GC(f&U@`_Sy^xt>@noOhJPyd&{-l8?dodX+Q8$~zj_30UvB@LU4^WDPk{Od;&qVRSixEAI>qmO z7Q#(dtl9KswpG4xYMm)xMcA@gw>mu)X0D&(#oi!7Z}i&nGPNP+ceytxrQjR}eTO-S z)Pc!t4k7J^(Za)p->f(v-pr{L@d;kjp$ICxDSh#?Xt=UhvK90_48-eDOHCvkh5vL+)apV@3n>d+?aNW>?mc;6_3>XwOwE%X54CVVkV>-_APz8eBLP!-TV@Q^?o z{qF3CO<{qTQu(ovU-g{wZk2q@TTJY{^mCClEZ~+a7STT2hjAEXg26 zP+C0^*5-hC4tyrYo3N|wT)n&(0|5JP2H{N^*COejigT;3VWAXsyrx$Zt}igZNe)mz z>sP(DS<{R&Mbq4Z%k}CuO59C9GTfI@HFwrhPp-bp*K&)&x9bEjguFHtKH_CtXS|U? z1Ekqp?!mm7XTRV9A5IEJ-!sc4xve4Mp?bHXQ9O#ak(`%+v*PE=a;-@XY#VI4u*Z&X zG!am^6&bt@7mA^Hgi$?a7+x$^z(X)fo>&PaA!Yj? z2`=Yx8{UAX+`Z5Hqgon2VNYM8u{mQETt9ce7N*ZAU9NqvWd_?+QUR|2j#|GaUyh@t zO7_DXB4z4CN94$_@!MKuf3bg$sRk1OQBo^CM{%gj7&+2mJW*$n>2MB`YBL+wOW!(Q zSU0^V^qa9C^sIPd(n+rNaNg>HC7d(wnOd~lM7B8i=RVu-$v955P$<@ujA?0vxMZoL zOUV$o$bBAGp7(6~PQ%OU2ma6j1h;frbp=)(SpF#D8e5(7PYF;qBAiW>dSBl^j?d7A>>6&{Td0`)mUqR*3=J7ti~e7!Yf473#|u2&Lpyb#HS5;ct1c&I=v;D>lLg3iC|$}psU_`bh5wcT6yze}K_ay|G;#zKhbiIf^E(bX`Hgoqga7 z#=SCXSadxH9Wwh8{R;a8IWY-ij%CAEd)nh8W|#=-SeA9Y^oeVdT&?%mY@!{2QY0kv zf^9o&4~(A1rJyrJX!@ ziau?mWcxjwUZllUbT?Y(`go)Epkq~^lb7L~7c5Eti{(X?boIT)K*)FaejUH2nuiZ9 z)eKqaawue5@ZMBe5k75l*%%g}lG)@>;v6vJ?2fbxe`lBufLe0<+QAdNkNm_gP4a;V z$iU(9?sJdDYbEyMf!@z`wXeMB$7;QKWv_Jxmsm?vV?eZ4zFx>#p?y2oJ_2;Se%0zS zyV$W&uK#9dJC3i~T@hXf^SReK^28AcePHbuwsgI-aKm8 zde3x<-qo=MkfN>48XGNM&j5<9pB_GU;aL1>#)n}xGv)KP`PQ~@kK~YUdiLFh0>y_# zGl67TX@OCoYx&g^)3*pbfTnMeR;l+Bff0=7-J02jf}U=JC0Osj;+8URn8<^@YAnyk#6kKqn4-(9><}iHGhALmdOpo@}R@aBa4> zSKX=Z=Uo>R5&A)z!6D6U@#-?(7Mt>d4AspDIw(j6cnc*?wJn8(r%NBPeg#}?FxkgI zi)!E``mi`IE3Zd0KeiqK1w?wvD|3DFp@{*up0xu}V~!I~Qcp z=|t5;bbN2P?Z#3QW>n(SI2+_|n^fhiCPSfbhcSg6XFoWNQNY@q1uw^`{7Yvb2L=A| zGpd5v4_&d+sj)LE=_BD+uFL3NHD@3A={jVDe~daVdr8rc+?P+lT4W8R89xp5equbh zPHTLpka}Px8GjQ~V?9PyY*eN2pf6+rBD&-PW~!EyI?o=EjpC606#X36_LE(7>`5VV z-d(J&`YP;MTY6|J|C%HxIOca}seWzPyM3Bz8TpLV$!W&Wg?Oq)7HhY1(4{ zO!1#<-Yl!y>-D`UN~egB(4p8@9kF;BXuRF}9;;+2c@uT2M#oEh#e?5%0JW?5)iRQ? zxm>siMmK!h2{ta2ayo&A-%VcnSAYaYb}OUJ8w#t)xAJs(6^rVHJM>>AUm0nX=pQ3@ zQ|#rr?);oLl`|B}Fv~A@G|Y;Q;8=>`W(F;rHYCQc_C>bg zsZ`Yc^V&HFuIe;DJE{|fxrdvkdP`zK0x_qK397SvSAik!1%I zR&VaGcJ;adX=fxC7_T#_y1`OP%uTd$iec#Z;pE5u4Pank9|8cI4-v9gtQ!3sO4wnyRwn=sHpU;v4YHluHLwTa^&JHD~8eTUL@ zhppqrHrg&`RWr}jGD;J_xJmwPV*6CUAm+M|&htr-l-J}4^%R5gVjln;OzP;QOBpJ= zTcB?}TE-$KY9j!Ejh42R*vF_Hk& zMAC81`Iif>SYarLQeJtE-AVtou_yeIRPA398023hF#NwHuvX`zodwh~G@tH`>Q7k3 zqilMO=V|~~ZPP{O=htB5C{|ij73emz(47D<`fz^FLdlwP=#ryFDuXF|CR7(P=v{45 zpaU^TA_D6|P)SSTlAC(w#LZA4^T3nuhQ>TR5SGq4h8u0cx5=?In$-54;vA?L<-ZiN zV7tT)ei$>#b@}xHx@+CDdmZ(6SK5KX!jExH$k7<42t_Zw1Bq=;7Pyy#638eLe6W7` zaY+^%u~%y`*ep4oM*Zoafz^m#vi`~+PV!?)cDtOK#K6iGzA>&I)GP-dGFNwb$`KbE zWcGljW82m&lND;zpt>V>_Qo}l2%K~BCY%d;(y~ zM4J~zg6zzsYAPZbc)qG8P_?510tW8Vb{2*ER&J0qZrHG$a3fz$zyZ6bV*UG9E4HTGVNpY%zZv9veUK7O|3|HATFWq?0xdsjN z3sJAdIyo_TS{oYnjLjB~<+FEmr}~uk@QhS1!wO9y;*Hpe9xHe?F})hOCO`Eirc{iN z*N18nQz#_b+@#9?D<%v#6vr#gyYT_<>u)clP_!Tk(vakr^#@;el|OOMb3!{V&iy)vd-qMozJjn*$r+zXORZ^zWA|DQ`@@4;UxP zMP!<$nwd>r>9_cOT)AUmDe|`0Z4TerkDyETE-pd(L?69x96}n4Au@odotgNBsyc)~ z?fs{iJ^z*}xeIPoFqa#<2X@zEnR`Wu;4{Z-GUDw0t(+f00AyW1i@X+CgA?iROp1-M zzg~GpH@XF94>hKJ=jXTTagRb1Srr!qTr!GRW7)$YD=Z)Qs7&=zNdl&e@PEq`awE+N z!?_~RLw{CE^M*i=Ye&162I6evaqw?SyUICFs!bD(y?*=g@ym*A%>CA=e74No7~ksz zdD0w3>i9s(i`Q_IVrj+srw82+($kl?Qfme#f$od~&ei134|STXz$S&2Zg0igA0iLq zl5R>4!Kh{a_K+_j0_@yqKjxTS& zr>b05^*ddfopgXyP~2n;ABl;0UlQi~JT;4(@ZY|H|LPw2Z${LEdP!X0;7=_I8OIgO zi*$m-$i>wi{JSS7$Z2#InvR;N!1qF4Td+yRedH4MM&zG^YV%=-@{^?~SEyK)B#B8Q zV?g)S_7Csfy|7lvyk&)i9b+6;g651#=Eoti3YAY<-cx)0Cf+|k9H|DN%ypoDe{9(I z)Hv6kFVT0Xv{da&6%rpeH4s&ZW|y{Jc4~@S+rJT%{S`EqNL>5EB&&0QgqXrU66Y)D zGhL5K!Z%-i){Htm3KK_)&QvP?1BHJ10U1U_GkWF}nj5Lm0pX%oi@Lh}vQX&ep20#t zMYEi6&NhvvFH9!BeWHez?(Ry6s5h^$@-`@Vt+gNP>J=A7W%bz+M#pahFI^Z zYQynHaM=#BQ=T&9bF*4TV89!P!=0A6MX%jATBE)saJGteNg)QUk*L>P8^-~cab{(I zGgmS(DuKnyM(o$eh$#Qu-xuGQW6pdkF&Q|UbP5!QU)Z`12=)(<<+v~Y zqasHr-t!uqgnq2SK7l|9T%}-|OK}-M(1p{6P--22i$ycTkLL&@nD6-hj(+d9F22)& z8isPSF;RqDLv?;lRKcX$0U9>9yg64mbsU-4w{v$i*`R$c7T1PWcJyzc7N+ND z3G3bS&hV5>dC|k9of(!r$6vf4k7`HjBxHpg3u0jWra6{Y>cOkF%%JzedrmxB&K(^L zEPWBOq*$M@Vtch;;?QuDIeP-CSdY%$%Sv}=+J1As3$B@VhK1=ilX4W^k3|{JtJ|%# zB2*MM5R;OfLJXN&Xjw?UjMAZ`jABYrddBgahxDdLKi~HusWS9;9ts}?iLhE2^%Tt> z=P0+~g>5|?VNDjAY6L{7mYQ>TG7?;y@=nO2*19^iNDXGiua~SC$3;!sAePWMF$wwBwk`fGQI4z00v%~dork0o60?u|iw_C>_ak0YKi={ndDIRLu z$+O*rJn(ooShfKqB_iAzy$F4^Hbv`t5@`^$~%cky}HpN72CAI(r)c*lsFSA~BiZ zT(;*|BPOY_wAZ`IT@*Otk>xahdGB7e!j#2~dxIKSJ~OqjAmVJd&oyFVw3oU&UQ?(_ zJjJO9G;vdS<3^C(1fiR{Npo4LR; z9372@)g!*z4o=NS^ zvhFfWNy)o3oL1a68Y1-fvc9AwEKEb75)C{YBu|^r&5ba0kj#ML;O53#T(~FVg7T@{ zPl2{S#rd>|VE{c0$1PKAN&U_*N4|4uN54yPbT`_vemad8RiZ@R(#gqbUkL%wTuG6R zPIz2Wld5P7ID?%Pzmnb*phklNkj5#EFIGM2B$|U^&BwHCGM$JE3Gu2ZQXIC)|6q+S zxt2QsnOqMeS4_%^kgBrj0Ipd`tyFqKBR}tzm=d*m1^VXU8hf0S^Jc$FLD7yLG#dF#C@7u&=;dDSeWQade`9x*B@2@soy<< zeoSxmUpKs#UXb(9YwV)oocKV?N$vd8|FJ z`ZHW!UK~ENY@EQ8ukcgwwHUa230vF8?Y z$cs@VVuQ4^H_Y>6cxS~#=jY{Cxwv*_88i4yyQl33_GcGyN9&G%UP`vf4d2@qxKGrC z5f-CXOwi*xDX(AO?X4uUSyJwDJ4U>F1%2wVEc$)F$D!rnfaqIB&uxi_0Ygpg)lt^p zC34;-Qkx{H@%mP;{&bh#bJ$r5z+_+{9Q>ILE{2ds4bn4tMMi?=#Q z1#~_4!+-n-vHpKLo}b#(En>>lN0dl^c*@2h zuhU{V#P!hgnU9S86TkT8v267pbAp;@UTLZB%64x$%_3D~(qh+7420Ov5O{N?u#0#~ z2Hp9UK4vHbisPfWOQ=>h<@G6nBka~VWV`G#f3-&^B2)m*cdc#=RkC1 zfT6P#C_n1b6S>fespEY=?U~5dSb+ZT^pM{$VQ_tuIw-pOLuSc%?r1)8j`w*9m;Tw!hSZ*3OS&)iH>M0JRHz`CZDdP6OH1l$oQe0gJSz5)_guB-0%mtN6z+!l73s~s1E?{q-b*MLLC zuay5$->7bctY1X3X>az!-4^YRb9#zKYbRf6ogfw#ZIF_;z&0EG`x}R(Y<2R7;}X+@ z{{9B=ZyVVOdm5>!2`aY)Ap-xOz|=gaSTAbry~IoQzFt*X=M#5q?CSwfK?9eD*a+Tl zffQw}`HwACnhUzwHc!y*lG?Hjf)bvRIOk(jQiLZT2P@n1CGJe($Rn)hf3)E?+*0IM zpcodnjqyD`ImYWaeAORVY6`Dyh*lR`~Laei~oqn|Se{AkPUjzb^TOfEQ z<&*GUjXQ(^LmRrfaxY6P^YH;PkrJUe7nhTyKcuzuT}3R!gI8hrn5jgD=dkma?W2y8OG&};%=I8b@%BI@G za@E2~jpkBr+v%w8d0a?~->5P^o@aEu+v0RT&&_*ii{H0#=Kq$FZ4Kkpxnv@<^SzFe z(@VxmT4ed7r6{Lbbeo6N@A`!D0|$ImC`|0B-f7JbmgK;~%+?Aq^>Z-p=9V^)Xj2yi zx^=@=$Oo4&nrp>21~Pa>!3F5Aax%rbX{8qH5#etORSlaco!nX>rcTL8Zf$}D_PUCzlQ%8W1I76yP z)^VF-C~gdTyl5uPQt&?)O&G*r+PVxGp^|InEaNx63v0wE->x_H^%McS}f&77jb^LWu37Tu=!XT!3y$z^D2&D^0O>>bPmI zAjZz|^n0{}UO-{lSmS8^I-fq)$L}oyJ(yITELe z%9u+#9T#6C*I6U}fW0k4p^Zz+n|`i&^dgutVOHqj{#AF5zhe+Mb-G$_U=l6in3$_D9zWzc(s9W&l;!-&wG;;`}b56W>rPWNa*Q=*XnI76l?110~z zt*o?0vEmQ}2}Eu{u@W+DLpKhH{8fbM&*X5317%c*fKNO9z*sDIv zlT9JI8s%&|YjRjYfl*ChNivi2R}LvA?+^OSkNIayh{pC4 zD-%`iFZD?GAZQb}ss(c-=knkswFv{3fXFmF;%dcVa&tR&t05&A(+doC!uXWitO?tc zoGV{@>=|6jg=DvX&BGGZ)VpqSM}J7~P`Q>Q z2;Z_KJ&DoZq`sSJ{6@NNpF09Onz7Ejg{r9n$}d^=Z^8KDtG)EZv^*+ffu70N^q|SC zLHisd(Ta+dbctUTzn+tCc;wwb3Jg{CI{Oxz&sb0!bH?5cB0k-;qv1s zWUR%~8$XQcHlqk0qK~Z6*9oTuR>-gem|z-?$F`eO<_C{a?)GG5sk%{|BNe`&wl3w~ zq1qS0zEuJ)-}#$s_#A1uJ++@^8i!8}A|i-zjg1?|gqCV;w)s$7Q$* zOY2FZ-BaGaNZZ%~4CZviD=9f3KSES04TEQFWzmBR#LTz0Ez8`frr#%J8c$Un#;5nv zQx#t)gS^frn&%z@WJN5L08!?WJ1b-3Qc+8Ihy8-ow-JCAVU2g{Df#h>$- ztbszJj5KtDym&PKCbL^ow@I4BH1{Di^#5^5@rZgMk~Z_{A?Hh#hxF0nsFq z4%xCr^aODx)yzZFyo=F{W~2O{g}z5SWOhJGCR2x7)OkB#=B_fBTRwujdG$$91;2Rp9y1!LB8y6^3a>>}QU;z6Y@Yb87gJhD%(4yMky>4`1r?HVDSPGKk--ru z&7@~PjWh3ClI(BWF)jF}4H*;Dr=dyj=iP{QmbFW z&2TF+;LoyyR_O~mYkJJlpkup%aHYZUBdr&1GGYCg7|VdToY14-+6pUoBOg*`P7)1+ zMy27V$rAMhua(C0+x*B#=h;l@pj)Pzvi$bM17~};4urkD6_s|@Q{xryCogB;Hnp9{ z<$Y81II`yrV^YU3Q%8&8?{(q*8l@*uqZv5W>SDiO<-+II{MX%7%axj4(uXqHWF88tqx6j#JgPy(u_$eS0^u+#xaZtt_c#D-Z?!-P=O* zDN=)aiVlyYru-#WkEoRkVR(y;m{Yz40kA)a{KKh25hUDNe>5aofAk;-wqeB8_)2YU zd&}c9?yNcByk?lABT&>^d$ZLr$f2?Z4QKQMdo zx1N+Ppa6aG-o{@mlk8!qp$*bL9285CVc z7Gi$&yZeXghFK$&6D&rr`@`B`(V-O7r`|8$&X$SP3QDY;X{zt4Q;Og5UZs%WH`7!5 zPY?|sIlC+GZTX(0^3A+hw~GxuEE?V~rV`%@y5X&Y9qSJtDppSD#u2>Z+hchiT&mvy z!|b_9FXkz1bD`kEIr=gLrCK4P> z=d7fTJ5%fn=y8Pqm)71gD6VLY7Cb<33k27Mgy0t3A-KDHaM#8)Sn%M%rEv`ojeBr+ zcX#*K_s);0H}mSvOx1MtzdpPA*xvQ+wbr*5GqGQ0t2)$GJOTqdh9A^hU&dk8T z#C;kJ&?moL&pzYkhI%@!*VO;P0a6MpH1^mU>EJ|x-uG1BtNPkgl?pk1ZOqTF@0|cq zK$Uc$(a=hb6W&Q~2jBF47sM^Sr8hG$B2=%$HB`Vmf5z)}o7QPdA5RQGVbDj1!Jiex zZa2-`FDaOlH-{q7o8r+BcnmxGK`*DOo;LxVU9Q37L%%XK3;O3;fA}3fEpOkSm=77z zgkb_v8+g8kZ&a{LbZ{Vh`Q1{6hP87{@4)|*a+l7nQ|?e}81PROr!8-G$ zN>IlHLx0fAJlfuV;=!?KZT*dAvRTH)>6@by?7ka)=ljtsyJWj&GaWm2E4%ycB=7}; znfr<>7A(+lfv+&0ko8~Jp6mKj+m#4?!a3@@sW9qa|)v5V`z2@TTlucAA-Tt!cvL%!(1^H5|UV*!;8)vWv;;Ht#5}ZcRgFgVJ+H{bpS8r$&l;fwRGi5pfE4DP+(VveEQk zG>7RU3}(QR@n=||QKE+^cH=(jGJuJH*1G7M(2G({?&p5hlf~s92-#15uIuC2 zI_rbg0H$}6Afs>kAXmr>=<&i@JlnaKh2?0s-j1vH*LZ_T+Gd>zP6MM%fE;l=P|!S{h?mT+*cqu z(yC#w^L|aseq>F?6>6#RYW1$9=B_`hJc6xaMaPtlEm6j6u4W;9zrwdh*B)Q;^6l;R zl)WO%`V9ZxKFu7P!V%&hGcSzKXIldW#+Bi6V^GpLemQ_5{Bi+Jg_?6GwN%Xh1|TH!_Jna zn9OQlW!251u}Uw+p0x{lE+(Xogwle>*Jv4R40hB<&7Ga8+pHzGTgJ=}K47S`u1tfT zh}u=`#UW;LtZskw+Uw0X;1b{}{c0x5Y9_>)?H*ubh!4_Go9$Ge*JGa9`)6rAbNuYk zZPEobS4QsJkT%Nn-QZa!(kk)G6Q%jp<546lWAIvG7{+V(+6raEK{YCY-~ zSkGz7lWfNdFDy*<_B*WG!0?95hxYiS^i8JPB85q~!q} z4%RW7i@@D157*mrOk4yYb&S=buD#Qcnax1ZYrC2Vh}7=A$W3)2=ah!>Q9ffK4;tmw zH=J?{8@V0j0u&`JleyQt>{LqoEp7?LSv#dLMOK4M3I@&IsYlO|hZxo{0D;_11EKi} zM+}HpU2lL-Nl&n5HN9|ZX4c|um&QuaTG@_l)XlG?{`v5Bl|`n1Rq&5hs_jXap&)rc zxrO-VYpa_Je^-(VHQ1k2kjs4R>ajZoV!sln1-rjUYzhUeQGi@-!6NrA^fwOX6p(8rPgr2xhLo_uuW4YxF=_eB1SC z;#q|_E+Os8XXx)!uw}8c=I)(6M`~tABJ?@^sB(7BML@QY(|bY=DQ?dZ15<9dO}(UKh}vk`4u@@-b_ zM@b==>V6UINv`QMkh$Dy4S(rSg!Gpzm3%%oouTHSiq;06Cxe9K0NO5Z+`%?Y)a zZ;B)jj1b!`RBPLNCy3QhxO-j8J2_J5Hh0`S@b)DX$mY|#mJ!-UWMl)RKSq15$v^ru zk##Ug4ZK}MAUWs2_`9Nd`HiEeY+@pV=mXB0$;v;4V@JMH7rmKEMPaQsA+#M?EjCr8 zegNM~F`%Ger^<)d%1EBCovN#`0wByBS+t6JRL_t}qjM*`2`@K;|Nhu{2D|iHzP}aJz^1+M;=&-ooSfA8O ze(NBuHA|Q)Bd7x8t|qk&j!iAvTFe(-tq6{yZMbo;NR#oTYoZ>P_U>$Wi%$jK0)H!* zK^CkHbn1*tXcjcRC|K&oFGahkZwoW;sh!jw%U+q;-dtRlD(ce?T=JKc_u;e39=>9S ztdXq>w43WVDj=SyFn6&S8fgbHbo5mol3%K@ZhQK>dT}oAA!=!nHZR!=%+FNS?+00a zvG~r77}Q$Xq+|uj0$VFIe11L^XiYO)1HCSt2`+p(udEqp-v6V(;oEJxrE$~yx!86? zOr_B_#@1zfmcO3)+lU~p?a)3Ak07ozmz~Ex36bH}hOO3UQo1%m2c3Y52u)O;_SuA~ zbII)X*}Rw-<9xa}9J5v8-c)=98*se5_NI969)3x1v&2J16BxNto6CXMlw?%bHOGLsaf?Z#k zx@pnWLVWwdZ0y|C=AO|~>=75v(1L}?@LdO)fU}(RW;%d4g!@=wPH%)nHHPH2_3C0_ z+V#ma0*jddt>cVt1CKx(EvXXSM&X%X$Jdda_DisNILa6fZf5NB^vUONzp=KrOi-A5 z>yrxz7^ihsjvGQhj>|in6nTdAnfHKKIHy3v3$^YJk*DEzJAlC4d!9BMq~W<-*hRIS zE=UvDR4^d+6{MnKFT?WWf3BOs<4-;i^^m@&C?IWQgKPL-izg+oIv;3jVyYjn8Rqy?PHD9Q-cY4v{@XpR z`tduikXNY*b;7#iEl>yP%Fu3G0(So-~$ zAzs?KJ^ZEyu><)*TAyZ(c-H9MN|MEz;}FbNy!;RoiS9a-*v@o54o_4q>W&;P)o{W# z6D@oLYx;_5x0pvaFf3K8!XU13(?}@%=hPo~w|eA(b9I+h+=JC$N;N7@;djH#tx72S zEfuYP#~S_e&@22`IViuEKkTpv3%K<}CWsW~n<^g+3`fb}+b-9z+KCpb$dKQrQ;=%l z;Jjuc^+8gJNP)jWHM4y&JM1NO4oJTgq>!*Uq%cnTeWSM6-A)Rca{Dg*j^o(@Ok9tp zyA*w>f{2+-moME}4Uz3%^<*^@Tg&lEfZ(`F5;GAa$!Oc;JGMk}`5{JIo1=H%k|>|; zZ-I)6=jn~IsP$H>*y1+-(6YwE!AIFXicwVOw!Gv)9+0f zivsU_eZQ;6UwiisY38gV_x^WY_)o|k9VEj{J&tox0uV>yPn-pE2XH~!HgrWcnfkf8VvR@1C1<29*U_G*_z`yg=P}Qt1V$K=$r=is~jh-e0ZROi9fv5QL;M}5f zRi%w$R(8nkXqj-*H#f_{)9EuD21Dm2U8o2Ad&H=9LKDPEr!Qi>?Xhf5D{#H#_Ds8M zQrH4$SsAM1uRm;thyp4cwtnU>DI}~}TD2@?!+B+c;SicJK|zO`7jfitPV#E|z)kOp zhhSU6&Vr1mU*>5}bzb)Oa_h%~%akjPG;Js#KCZ{Brg-(`rJn)Fa>m(K4cUe3*H)I517$Y+jg2(`%9fpW^1n{p<#8Kro ztLhe@l5FAMn@uUYxq?NVc6Ruf?XzT0*%am`C%3M_fnYd9o3|jo11JC9jh`zN@105j zAw6}6SGzHIqciemDCP6y&pS7C=DYxOh=ZkH-mltwAkWLeVq(pWXDi>d#0nJNea=u% zsbMH@S^8n`eJ&K*5LC%4Gz*Nb1;OGGr4e~qd@a{GboUdriw-_#Xnq*M-hc=b(6GU0ghd^(8h^jF8AVr>lgo2rx6%|+Ls6~t_Bn!mzH>k7ff(yj= zBAc<})TQi3xx8XnBZ7?Y+yt0smvSu&v$3eYyK1WsoU8rE&4XCk-rjGkgp;8Rv$7_K zknOy3AWgZW?P%zae~O>blE=@pej`=N;rXY{duRqcK#3Qj16p<)NTJK>d6J-4ca~9O z(A_|)S|2Qwsnn^MD(Hp~Vm9dlm7a2cc3*F|Kf+|g0`=<*v4Is7&O>b~8Wp9f`m{z^otRbK!wdN5Lv|Rs%ukH z2&{v{2gDdm6f$xgKK`t1b@SXLIvAmQ>~7toEzZghP0<k2pZhfnKq9^m-=EIxz1-6n zKK(NP@@tq6_b8#^{LCQf)}`nJzcbQ;i%rwiLC^M&z`%8OrH<|5PJ0fM(e#Yf{=UB? zY3mH^gDn$(_QBrwl`9Dshb7JnbrfcXK5FCFxJ;mrl=w^}|ln%Z$+Y(y)Z5SG& zbH{LXV+t*1{4hp=JDOM(#O2lmh^yPu`$%^V1TA%^f|d03a&G4)8$KIly=0rnCB6BR z_JSKRlWXbsL5cfD*N=M!|1FSg|MwNq*~5y01n-j^1}dqbZUzgL+F)(I(@T;~=FaWD z0kb91j?V5`k0_6xhr0w<_ypab=yRuh)8hU&w8j3kaz{Is&=}!b$!x~GJ$W=q2L-jap zRcTeZeiN&npw_S-hHKmZR$qxU}7D0ZOOM*l!&EYKNyI- zWWqE&wQiFx5tD-}HnZ)@<2*(;Ik8GUN-#?wPqT*ODmj;E7eQnD7J;A8I3ZSH!CQ6^ zTT{TpSLElSL$kwS-K)yu_OhuwV$aub;5cM_jXaRAJuB2EcQV25m7v|x2Tl8=GyYi; zFK3;WzlXbEAplBPfSoEaWdXC%Q{c<#S#X>Zr`W^hYX>=80ICx-9{O^0Z(Ct zfKn-6wbLyYupU~W)yrGURXS^`z1#D-Zr3rcUhJV_ zp44>L(mPjASA!&iGM;=Dt*I}3Dy1g3RsC{r9mhs>mzBSCEbKktd zs2zn724E2*gbC`~Bu#ABW_(~pC%}6q(=*U5r@S*x^Uw!t_;~C-@~`PD-??lWBY$n_ zr=znvYIoqL5ctdQtT?-z8E{0tSuPN(R$>Hz)g8AL7fR0*91tQwlip#IIKIB+@%nS~ ztzC%O^rLjm#*ad+xn;Nb0&5jMDgdxZwV&Y?ht1xB-SV1+>DQTVd$80Gd^g{Ox{e7TQNt&8Q!mfO*?XFDd$6~$*(9v}L{KnseIGn3Fm9LQsMApB zlaX2Pv(h321DwBnj#pOs4O=%fCAh-E(`v!uu;eS$xF%XF6^;itu-0$m~ga|5wG__@S6-d ziM8|jddpuqakMqjwxxSCM*3}pR;k-e3~76*nr#=;&1C5cO^)cQ``8yeI~t8>0dr1O zK2N|AX+EW!RCOvOx8!jfb&if4FUmKKf2Nu7MEwrvAclJE=JS@ya;FTorjaI>NiMNV zDJfi$XUfX6GVC+H46tRa)TgzZtQMTOnZWm7N!x*lDeOwwTwh^PNc&P0qS(jn{MC&G zqKf7$7AjhNg}ht}EC*M;q^dcW5D;uL_&ARwgNqBLV)Wy%ce=qhGl(2^v{NvKID!XM zPtD48AUlDsN9#}$sveZVy9{4eK7oJ1uWF#(^_=QxXMdlFHt#trGQX8<@P(H2XZKAH z77^>YO389_>WSJO%8AsVZDj_-JzTXqkQ*3PN99l zwedRZ5I#8KAekh3Ah)^qepZy)jj9<)nC+8WC7or_sSA<#WAWuM)9z4%K0-y$Aya^5sw*nJf^%)LSDjzO&!@*^}#}AY0B+}Z?V!P-(4j$5zCVQE)KB*bDH>#92 zxrwCne;}T{xeqK60!x)nVY&~UhEuSuE;`r8DdbGNMHu&NjK48~#a#Y}cKCAXNOYUg z1;44Y?wlD3rWXdCt~{>e2hB}LUSz%XGgQx)Y!A{7D>9Ktwe zC^wh&co%N(4DGoepG!(w>C9FVsO8gGeCMx(H-wfc(8b8Bm2k9gbQt%njUc4r5jMCm z7Y_><+rDnlmSYLkK*DVMPjY>#F0$S4K24dj`2;`TrU{(Q1M5-O>uQ6Uz7+TQGig2$ zUXLNOU9GRKB5eA@ZRu7Jyid=D1^eQ&hddGV%ocSvUII~vw-vEqxXOp&(ecl*+k}IFMgGKZ?f2-h zXX<4PY%cMyMQbL8k;ktbCw{Qm)ZyrTWzKyTI8=Akv|lO9Q8BsNoAOE*j)VgrM&6VK zL7}77LF8<2!!x;eP>=q@EgD$Yz$}*KjgBpfO)o5 zd#z#ayX*%gP6NL>S=>Y3bXpZpuh*E84-LaLT1~5wE()W4yz??T)!OI){T#rQsn@pzrkz-G>c+5;aHZdjuFs9*olz;MhYx9R+6_syhh8Ddq!`rdCva=a9%B|JuvbGuUU{TPIY$) zl2jxO);SPzzUbkn!UMHedByrrsN2jrwN^s(ciXx<0~|B$4wJp-FY0w%jR~f@!HQ8D zpSasg)4HYYk2&ao^!@9X3DO1Ah`V56FRcPsR8)4ovh20>n|ENU zKNUWpF5Sdc7i46s4DJndVtlT=rwuhO#FxanJ1e5=#Hfa&O`o+BtAg>dQtof)QT$)^AHo>8 zQ4kCvQk>noL)xu^e6&2KC8Px4jn}%GC1v?~eT^J7f++M;&CdIxYYp%Zd>?WYknjsMY z-yL|!YI#5@2ju7TQ8a{H%=-&#(2YpH%+jZ#2KMJrx8B6L#cy{6?-wzBKu7Rd=p>Cf z3cvoexXDlW?19P=LG$^KSG68*bK`+B-9pWf(aMy1xuc8A~WM zkST|tw_cO;@{9b>eVzOG56N%r)(b3GDy@n88&c_h+F8*be=#=7YTbk>^besw4i48z z$XHSw{l7-ZTOY_@>pvarX*@ItPLNo{dSm^qE!t7~PP|uq zKo}8>uP&?OCe{Dp;VGP}gq3m-L*d2MAU&RR!{)IyzxZ_Vp}Tkq)ru_D{2-#yPDN0M zquXG+Cs^L6hQj;_e?bFnzW~ka60P*wanw1QG=^;Q0IRbE?n;G=PPvcs^w~uCR&FR? zA^e1o(s2Z~YFyaa`?j|BC{NzE0lKb!#XK5hqLssxf2)IZ{xG8Ze13)(hy`%@IxEkH zSK21Hc>F6KLNRgU{)q*C?pDuVIxP4iO;2vLQh45zmEVW>R+savEBrxY|8#>r0gyCLjppG9sBss{`1>vgdV7O{hNnnk`n?`{wLl%p%AdZ9cyoMiHI+@ulqh-4ln{ zpKIT|DRq;tl*wu20`wfArWPy;O`Qo>8_ zl-@4k9%B50n#Y^AQ@eb`1>d(KGd5MK8Ul+IPxED0G zCr5r%xXWE=vmlER)L=0nD?z!UuBWZ4wq#yT&%^zkrEwyok(M-{z@t3&v;m9O`C z&;4;%%M48AeZMd0*aZ3q&n3q$`h#e0c6(|7mE&^n z$N4HBZnveNW1r<}sO0rB5~*4#l6UP=H*E&(7W}zkO6Bw4Il|oadT3yZp@xd>>BXK# z5wP=87P^n21(eS%kE9fi%2cWk?(8{F8cT4m4+s>M@|^H`3Qls%t>}Dh*QRMtB|jTQ zIK|7`o834etjhTW&BIYlP4pZ*-YMoB%fqME0cM(3iu>t!Vq|UI<-0gne&UO_9U-mU zgj*xjKRu5~8GXO00?AC#HJW9JHkbLVt+*mZt|Z{h5=}S4u;%Kk;F86&E%GRYAsuk+zBc*t zrDqz<{WxRiq%6GOFVL)FMYK4mM$7mfhb@v`3zF+=cWAN!X$gxzo*o~)4+k4lE$UQs zIc2umjp(}Rt%f9~jn+l}Y2U56Ycz}r*J=HiMJW~j%(hjaavu5KC|()#0+%kqdE0jVObc^WD}`bhtr`i=IRP)JT#PiHMcQ%B711a3nJ)pv<6>}qBe{{f zVyM|k$1)@3%uu6iem*Pbbj;?B%;=Z5dALT%rAnwu!&&@6`t#@9!m!NsKuFnCTi%^S zCFYR{c0Pts1PWl-m|6F7?PVaR)rJB!{68Vpp9pSFZfnNI#;5R9bfkE4-xET-dsy(e zf*~NPB#~d#9RGK#eV)m>x4SXjD#k7zY^bZ4&SlY4kFs#VBrvdp0u4&#;9w(K4uT^k zQS3+P0AX|0;^3W*7Ei%AybryQn{s%Di!J zjqM5IUw?c@NM5nYN|~b6{PKbdRpk3IxeKSYw)J>8Jkx)!h#gi+WtGC_gP-3k z;+vD_mgLcS3>pHWF_L;2!!y_W?D!JvYSxt_f=&b-qu%b7ND>{2KC2c+CqHZUbE{$D zGE4h7x@C4TZ~s_XmaGH}ZavDg8UlqqWGDAl708FXv<;iUY2fPhQLg+L2;?y?ee>T{ukwUTp|Cl@S(=+tdRm!|hDEHM3ffm2kCDmx zWV_s^`mq^vo(J6p&@>|469N9(!f+vTy^oAqYh)9BHcT{Hp{>BSx_-NOcFwd^? z1O0HaEC}}rEgFNwIQ}sj10bVSyMxIL^e=zsj~6(K)Zy=!*82WQ%tSGn?6vyZy?{%= zN9Nbv_QyN0biOHBM-d-LH3tsxdYflQW?eyCZqoP78baVXBkP}74S)~5NQ>xY077k1 zqrveiHUC$dfDo8YkDnGDCfwMgvbx45NaWrdXo<^Ds^>ny^Iw`I($NMO{onQ>Fb0-- zH&`I8&WC0MU>!1|-)t5@-k_9IWK=A@dj7TZ7a}p3EAfyuvR*HKLd#goU}IZ?Z5q1){Fbqq3Kr z1Pfwwj;0>mG|(F(I)^`Z{q1#xS&qv)&xe;2*)>OO+c8W5(}ge|T~k^?Ll4d*crxdf z^Vn5n+c_nbCuI!YOq7^4VtL;FIt&%iR8160&kYSB8F)b*kn`iub`-A=Z0au)qNVMnb|1`by9yrCUz|NsDUIC7L6s+};j}H=J zBN^gow^w_@K5f@fOK>zgo-$VdA^7u>xgg6^dj$+^pd5Y8J5eM~WstDyOuV0iESbGS zfBTMcPgXQe^gQXGutsc?=~^+}oSK}i9y0iHT5M;zNa-~?+ue7Q)b^NX^|f zmt`xnu23r)G9-D*U`@@s@E7Jb_*s2HU7MC<$JTR407idO_lC#F%3s#AbXDJqlM?c5 z9=GB?B!YzAxa5*+eWVaZB8R=2=9JX`nE>LyeH$idE}IhA{=btk9%k)Z>hv6G#?03l ztdm)zpG>BzcH|2`#7CfG{G1boi7!NGU--@K>KLrHx|W@L$u>_<4Q-g8C-cfwsqHy1uF;WQEgC^ z?-wHI1|??RU4$MOT`xXq{713!YV*HmW-8r2liu{@sV%wrWw43OkGz5S?V(s_MZ&L7 z|Jj&!3V28CFcIrJOm?2cRwQU)vzztKq4Eu5Lqt3n=Y4Fh5`>cuGUmBrFr0Q$d^wpq z3Wd?}`paY zC;wc)Tb0+4&g{GqO8&s5s0Tgled?JHmNNSHH=@%78^LrGDxF z%ZKUyxkhYe&ZCinyRKr0+=VBbBg4TI9`@?Kicg3?4F%~CqDc05?+4l+rl0{kwRh0q z*}8zTLnVYeQINgGub!5CIZR`^?R!fy1x0c#`<@r= zok2NaEa2ngmRTE8SEnHkshSQZ+C!fSAbe!<^>-R*{bwo+EM$7)@JE_T;b~sqSE~cg0=^;EkC{@`W zzB51;{#@5=0w)KrnGIu;9tfs@3ufyL5znl-l5Cnqdfo2%)1Ik;cI~z`@p3}Y8!!zw zcYR+jpPz!uwUB=)YU`!zkd{cW(_0v7yZBmRGq=c>e>I$UQ754$fMrZ`piwM{WG?=) z@&sZe9$yn^HJ9-{c^~oIH_DDN-xx&y=fE2A+W63#-BrYc-AkT_5ksw9U#AEj@Ei^= zCyNtX8e~8@#O%yUClDF3dRT>KOth2YaEID8%c8gxEV#N{9gq(6GC6*(fo&x?P;Ihy zTPO@JYST0*LmZ;qoKhswX>ReWtZz=+n|a4E^q;grc|ke*fLx`574o%}05=f#eevR zXXOpjVw`HGcLrcDESgItDp{Eaw-2QP^j-3Y*|J?y?3CbB|0gheMqss&Pf(v=zSjIE zTAQB|Nx$vKKprH|8(jz;`8?KMO0LA1x-!`Elkdaas8!VAmL=k6yx#nW*(C2|XWarb zptrM*uqC0RBypg6_RQx0gyhFL)Ffs}l`UbsE=tfGe z6h?oiQqPEqEmfVs^p8Y7Yq}X)p@ckoy;5-hIE-aaB;&R}zi?w=Q1XS&C}L0G{ln8M z{Q}bz0Du|e)sAT9Cc397AL`Kj3ofw}Ybwp>K9>uFy)8T{YLNjjeG(y`PY}J3M%!DX z?{jV1(H5`q>LVrqa`mu4iMeA$vV^Nu< zeF$r3tQyWs=7twKZ=u03a2Pp1=JedMf?ExEkFw-_k2u2erwJ zH`jfvM%`RM`u6n;Jf7%9VvS!DVl%&}ZPUCEHpx9NVsz?6Esvv(W@` z2oc~fwscc^iV2JlM&L|oU~p&UANTIQd^fam`c^XACkyo;J*)%i?AD z#-}l`OjxEZm4pd?`}TDhzJwSBx7%d~_pqezW>5z+Py#Z^S%O}9|8V){anTOr|DaUx zE1fOQtzL7RwMhLou&&`Fdj7iX-4c$VgxP0hOz3h5QU|H1;&D?B zb;Cw=lO?C4<|;VNMS+0l4rwqm(G9~RyxTI`nwm;~n}ijYHW@o?fZS&yD2dO0X~FGC z(V?;fLzF*ktTm=}=h$CQP6wmt8ti75-?a1|KWi3Wqu%Rye zp5;uXUHej*1uTD}46onW3!u82E-z1{etyC<-Ddq1yEzM^-MmJ?F>>VT%tTXg`){>G z__d<^rKn;k-PgRV{{=O!o*B`@8vc^n%`k#obU~o{2|%C`Pl+}H{>|5_1rYW|(ufsvz##kq-O)BG#& zXU;qoX6DoqktKOoKigP2&aJdbbCrv~HJYAlN zlr{!sCktz=`1!)FJZ=tNnXmWvdj{-XWuMM+LA+mLQnV?ID$JR@pq1p+$$54g$WC6T za+1y3^2Hti+6=NqW+t*EiT9fyWwWCH`X(?_fBO*$ARvhQDl64 zUK}FMp<{cMz@+dM>8HE?P_viu1ve{!?Om13iMcg!0%u#W~Q=m4@J;256q0kke3!(+~O!~LMvQ%sT&-4=)f|#|Cf4eDD-^x2FKZ* V+b%kpPX|ChQeyI=72gei{tsZlJYoO< literal 0 HcmV?d00001 diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 5f5933b47..3677ebe89 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -239,3 +239,52 @@ Possible parameters are: The fields in `webhook.webhookstatus` are used for regular status messages (Started / Stopped / ...). Parameters are filled using string.format. The only possible value here is `{status}`. + +## Discord + +A special form of webhooks is available for discord. +You can configure this as follows: + +```json +"discord": { + "enabled": true, + "webhook_url": "https://discord.com/api/webhooks/", + "exit_fill": [ + {"Trade ID": "{trade_id}"}, + {"Exchange": "{exchange}"}, + {"Pair": "{pair}"}, + {"Direction": "{direction}"}, + {"Open rate": "{open_rate}"}, + {"Close rate": "{close_rate}"}, + {"Amount": "{amount}"}, + {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"}, + {"Close date": "{close_date:%Y-%m-%d %H:%M:%S}"}, + {"Profit": "{profit_amount} {stake_currency}"}, + {"Profitability": "{profit_ratio:.2%}"}, + {"Enter tag": "{enter_tag}"}, + {"Exit Reason": "{exit_reason}"}, + {"Strategy": "{strategy}"}, + {"Timeframe": "{timeframe}"}, + ], + "entry_fill": [ + {"Trade ID": "{trade_id}"}, + {"Exchange": "{exchange}"}, + {"Pair": "{pair}"}, + {"Direction": "{direction}"}, + {"Open rate": "{open_rate}"}, + {"Amount": "{amount}"}, + {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"}, + {"Enter tag": "{enter_tag}"}, + {"Strategy": "{strategy} {timeframe}"}, + ] +} +``` + + +The above represents the default (`exit_fill` and `entry_fill` are optional and will default to the above configuration) - modifications are obviously possible. + +Available fields correspond to the fields for webhooks and are documented in the corresponding webhook sections. + +The notifications will look as follows by default. + +![discord-notification](assets/discord_notification.png) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9fbd70e42..18dbea259 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -336,6 +336,47 @@ CONF_SCHEMA = { 'webhookstatus': {'type': 'object'}, }, }, + 'discord': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'webhook_url': {'type': 'string'}, + "exit_fill": { + 'type': 'array', 'items': {'type': 'object'}, + 'default': [ + {"Trade ID": "{trade_id}"}, + {"Exchange": "{exchange}"}, + {"Pair": "{pair}"}, + {"Direction": "{direction}"}, + {"Open rate": "{open_rate}"}, + {"Close rate": "{close_rate}"}, + {"Amount": "{amount}"}, + {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"}, + {"Close date": "{close_date:%Y-%m-%d %H:%M:%S}"}, + {"Profit": "{profit_amount} {stake_currency}"}, + {"Profitability": "{profit_ratio:.2%}"}, + {"Enter tag": "{enter_tag}"}, + {"Exit Reason": "{exit_reason}"}, + {"Strategy": "{strategy}"}, + {"Timeframe": "{timeframe}"}, + ] + }, + "entry_fill": { + 'type': 'array', 'items': {'type': 'object'}, + 'default': [ + {"Trade ID": "{trade_id}"}, + {"Exchange": "{exchange}"}, + {"Pair": "{pair}"}, + {"Direction": "{direction}"}, + {"Open rate": "{open_rate}"}, + {"Amount": "{amount}"}, + {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"}, + {"Enter tag": "{enter_tag}"}, + {"Strategy": "{strategy} {timeframe}"}, + ] + }, + } + }, 'api_server': { 'type': 'object', 'properties': { diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index 41185a090..9509b4f23 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -1,8 +1,7 @@ import logging from typing import Any, Dict -from freqtrade.constants import DATETIME_PRINT_FORMAT -from freqtrade.enums import RPCMessageType +from freqtrade.enums.rpcmessagetype import RPCMessageType from freqtrade.rpc import RPC from freqtrade.rpc.webhook import Webhook @@ -33,46 +32,26 @@ class Discord(Webhook): def send_msg(self, msg) -> None: logger.info(f"Sending discord message: {msg}") - # TODO: handle other message types - if msg['type'] == RPCMessageType.EXIT_FILL: - profit_ratio = msg.get('profit_ratio') - open_date = msg.get('open_date').strftime(DATETIME_PRINT_FORMAT) - close_date = msg.get('close_date').strftime( - DATETIME_PRINT_FORMAT) if msg.get('close_date') else '' + if msg['type'].value in self.config['discord']: + + msg['strategy'] = self.strategy + msg['timeframe'] = self.timeframe + fields = self.config['discord'].get(msg['type'].value) + color = 0x0000FF + if msg['type'] in (RPCMessageType.EXIT, RPCMessageType.EXIT_FILL): + profit_ratio = msg.get('profit_ratio') + color = (0x00FF00 if profit_ratio > 0 else 0xFF0000) embeds = [{ - 'title': '{} Trade: {}'.format( - 'Profit' if profit_ratio > 0 else 'Loss', - msg.get('pair')), - 'color': (0x00FF00 if profit_ratio > 0 else 0xFF0000), - 'fields': [ - {'name': 'Trade ID', 'value': msg.get('trade_id'), 'inline': True}, - {'name': 'Exchange', 'value': msg.get('exchange').capitalize(), 'inline': True}, - {'name': 'Pair', 'value': msg.get('pair'), 'inline': True}, - {'name': 'Direction', 'value': 'Short' if msg.get( - 'is_short') else 'Long', 'inline': True}, - {'name': 'Open rate', 'value': msg.get('open_rate'), 'inline': True}, - {'name': 'Close rate', 'value': msg.get('close_rate'), 'inline': True}, - {'name': 'Amount', 'value': msg.get('amount'), 'inline': True}, - {'name': 'Open order', 'value': msg.get('open_order_id'), 'inline': True}, - {'name': 'Open date', 'value': open_date, 'inline': True}, - {'name': 'Close date', 'value': close_date, 'inline': True}, - {'name': 'Profit', 'value': msg.get('profit_amount'), 'inline': True}, - {'name': 'Profitability', 'value': f'{profit_ratio:.2%}', 'inline': True}, - {'name': 'Stake currency', 'value': msg.get('stake_currency'), 'inline': True}, - {'name': 'Fiat currency', 'value': msg.get('fiat_display_currency'), - 'inline': True}, - {'name': 'Buy Tag', 'value': msg.get('enter_tag'), 'inline': True}, - {'name': 'Sell Reason', 'value': msg.get('exit_reason'), 'inline': True}, - {'name': 'Strategy', 'value': self.strategy, 'inline': True}, - {'name': 'Timeframe', 'value': self.timeframe, 'inline': True}, - ], - }] + 'title': f"Trade: {msg['pair']} {msg['type'].value}", + 'color': color, + 'fields': [], - # convert all value in fields to string for discord - for embed in embeds: - for field in embed['fields']: # type: ignore - field['value'] = str(field['value']) + }] + for f in fields: + for k, v in f.items(): + v = v.format(**msg) + embeds[0]['fields'].append({'name': k, 'value': v, 'inline': True}) # Send the message to discord channel payload = {'embeds': embeds} From 4b70e03daadc8cc3213656c6d8eaa32815096dd1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 17:45:37 +0200 Subject: [PATCH 074/162] Add some rudimentary tsts for discord webhook integration --- freqtrade/rpc/discord.py | 3 ++- tests/rpc/test_rpc_webhook.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index 9509b4f23..5991f7126 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -51,7 +51,8 @@ class Discord(Webhook): for f in fields: for k, v in f.items(): v = v.format(**msg) - embeds[0]['fields'].append({'name': k, 'value': v, 'inline': True}) + embeds[0]['fields'].append( # type: ignore + {'name': k, 'value': v, 'inline': True}) # Send the message to discord channel payload = {'embeds': embeds} diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index db357f80f..4d65b4966 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103, protected-access +from datetime import datetime, timedelta from unittest.mock import MagicMock import pytest @@ -7,6 +8,7 @@ from requests import RequestException from freqtrade.enums import ExitType, RPCMessageType from freqtrade.rpc import RPC +from freqtrade.rpc.discord import Discord from freqtrade.rpc.webhook import Webhook from tests.conftest import get_patched_freqtradebot, log_has @@ -406,3 +408,42 @@ def test__send_msg_with_raw_format(default_conf, mocker, caplog): webhook._send_msg(msg) assert post.call_args[1] == {'data': msg['data'], 'headers': {'Content-Type': 'text/plain'}} + + +def test_send_msg_discord(default_conf, mocker): + + default_conf["discord"] = { + 'enabled': True, + 'webhook_url': "https://webhookurl..." + } + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + discord = Discord(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) + + msg = { + 'type': RPCMessageType.EXIT_FILL, + 'trade_id': 1, + 'exchange': 'Binance', + 'pair': 'ETH/BTC', + 'direction': 'Long', + 'gain': "profit", + 'close_rate': 0.005, + 'amount': 0.8, + 'order_type': 'limit', + 'open_date': datetime.now() - timedelta(days=1), + 'close_date': datetime.now(), + 'open_rate': 0.004, + 'current_rate': 0.005, + 'profit_amount': 0.001, + 'profit_ratio': 0.20, + 'stake_currency': 'BTC', + 'enter_tag': 'enter_tagggg', + 'exit_reason': ExitType.STOP_LOSS.value, + } + discord.send_msg(msg=msg) + + assert msg_mock.call_count == 1 + assert 'embeds' in msg_mock.call_args_list[0][0][0] + assert 'title' in msg_mock.call_args_list[0][0][0]['embeds'][0] + assert 'color' in msg_mock.call_args_list[0][0][0]['embeds'][0] + assert 'fields' in msg_mock.call_args_list[0][0][0]['embeds'][0] From c9761f47361203eafbb08f9a5413e88e0e80159b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Jun 2022 18:02:03 +0200 Subject: [PATCH 075/162] FreqUI should be installed by default when running setup.sh --- setup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.sh b/setup.sh index bb51c3a2f..202cb70c7 100755 --- a/setup.sh +++ b/setup.sh @@ -87,6 +87,10 @@ function updateenv() { echo "Failed installing Freqtrade" exit 1 fi + + echo "Installing freqUI" + freqtrade install-ui + echo "pip install completed" echo if [[ $dev =~ ^[Yy]$ ]]; then From 56652c2b391fa1714bf706ed156df72910b7dad5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jun 2022 17:09:47 +0200 Subject: [PATCH 076/162] Improve test resiliance --- tests/test_freqtradebot.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index cd7459cbe..7f9bc6a46 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -210,13 +210,14 @@ def test_edge_overrides_stoploss(limit_order, fee, caplog, mocker, # # mocking the ticker: price is falling ... enter_price = limit_order['buy']['price'] + ticker_val = { + 'bid': enter_price, + 'ask': enter_price, + 'last': enter_price, + } mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': enter_price * buy_price_mult, - 'ask': enter_price * buy_price_mult, - 'last': enter_price * buy_price_mult, - }), + fetch_ticker=MagicMock(return_value=ticker_val), get_fee=fee, ) ############################################# @@ -229,9 +230,12 @@ def test_edge_overrides_stoploss(limit_order, fee, caplog, mocker, freqtrade.enter_positions() trade = Trade.query.first() caplog.clear() - oobj = Order.parse_from_ccxt_object(limit_order['buy'], 'ADA/USDT', 'buy') - trade.update_trade(oobj) ############################################# + ticker_val.update({ + 'bid': enter_price * buy_price_mult, + 'ask': enter_price * buy_price_mult, + 'last': enter_price * buy_price_mult, + }) # stoploss shoud be hit assert freqtrade.handle_trade(trade) is not ignore_strat_sl From dff83ef62045c2b006702d5bc855cc9051a3bc80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jun 2022 17:30:01 +0200 Subject: [PATCH 077/162] Update telegram profit test to USDT --- tests/rpc/test_rpc_telegram.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 11a783f3a..355a8b078 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -643,16 +643,16 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: assert str('Monthly Profit over the last 6 months:') in msg_mock.call_args_list[0][0][0] -def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, mocker) -> None: - mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) +def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, fee, + limit_sell_order_usdt, mocker) -> None: + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=1.1) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) patch_get_signal(freqtradebot) telegram._profit(update=update, context=MagicMock()) @@ -664,10 +664,6 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, freqtradebot.enter_positions() trade = Trade.query.first() - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - context = MagicMock() # Test with invalid 2nd argument (should silently pass) context.args = ["aaa"] @@ -675,15 +671,15 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, assert msg_mock.call_count == 1 assert 'No closed trade' in msg_mock.call_args_list[-1][0][0] assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] - mocker.patch('freqtrade.wallets.Wallets.get_starting_balance', return_value=0.01) - assert ('∙ `-0.000005 BTC (-0.50%) (-0.0 \N{GREEK CAPITAL LETTER SIGMA}%)`' + mocker.patch('freqtrade.wallets.Wallets.get_starting_balance', return_value=1000) + assert ('∙ `0.298 USDT (0.50%) (0.03 \N{GREEK CAPITAL LETTER SIGMA}%)`' in msg_mock.call_args_list[-1][0][0]) msg_mock.reset_mock() # Update the ticker with a market going up mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up) # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') + oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, limit_sell_order_usdt['symbol'], 'sell') trade.update_trade(oobj) trade.close_date = datetime.now(timezone.utc) @@ -694,15 +690,15 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, telegram._profit(update=update, context=context) assert msg_mock.call_count == 1 assert '*ROI:* Closed trades' in msg_mock.call_args_list[-1][0][0] - assert ('∙ `0.00006217 BTC (6.20%) (0.62 \N{GREEK CAPITAL LETTER SIGMA}%)`' + assert ('∙ `5.685 USDT (9.45%) (0.57 \N{GREEK CAPITAL LETTER SIGMA}%)`' in msg_mock.call_args_list[-1][0][0]) - assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `6.253 USD`' in msg_mock.call_args_list[-1][0][0] assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] - assert ('∙ `0.00006217 BTC (6.20%) (0.62 \N{GREEK CAPITAL LETTER SIGMA}%)`' + assert ('∙ `5.685 USDT (9.45%) (0.57 \N{GREEK CAPITAL LETTER SIGMA}%)`' in msg_mock.call_args_list[-1][0][0]) - assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `6.253 USD`' in msg_mock.call_args_list[-1][0][0] - assert '*Best Performing:* `ETH/BTC: 6.20%`' in msg_mock.call_args_list[-1][0][0] + assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) From 7619fd08d65e4cafee6e5a9f227987392dfe8fe2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Jun 2022 19:31:32 +0200 Subject: [PATCH 078/162] Update telegram tests to use mock_trades --- tests/conftest_trades_usdt.py | 6 +- tests/rpc/test_rpc_telegram.py | 102 ++++++++------------------------- 2 files changed, 27 insertions(+), 81 deletions(-) diff --git a/tests/conftest_trades_usdt.py b/tests/conftest_trades_usdt.py index 6f83bb8be..cc1b1a206 100644 --- a/tests/conftest_trades_usdt.py +++ b/tests/conftest_trades_usdt.py @@ -95,13 +95,14 @@ def mock_trade_usdt_2(fee, is_short: bool): fee_close=fee.return_value, open_rate=2.0, close_rate=2.05, - close_profit=5.0, + close_profit=0.05, close_profit_abs=3.9875, exchange='binance', is_open=False, open_order_id=f'12366_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, + enter_tag='TEST1', exit_reason='exit_signal', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), @@ -157,12 +158,13 @@ def mock_trade_usdt_3(fee, is_short: bool): fee_close=fee.return_value, open_rate=1.0, close_rate=1.1, - close_profit=10.0, + close_profit=0.1, close_profit_abs=9.8425, exchange='binance', is_open=False, strategy='StrategyTestV2', timeframe=5, + enter_tag='TEST3', exit_reason='roi', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), close_date=datetime.now(tz=timezone.utc), diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 355a8b078..48acda47e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -679,7 +679,8 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f # Update the ticker with a market going up mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up) # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, limit_sell_order_usdt['symbol'], 'sell') + oobj = Order.parse_from_ccxt_object( + limit_sell_order_usdt, limit_sell_order_usdt['symbol'], 'sell') trade.update_trade(oobj) trade.close_date = datetime.now(timezone.utc) @@ -1235,71 +1236,43 @@ def test_force_enter_no_pair(default_conf, update, mocker) -> None: assert fbuy_mock.call_count == 1 -def test_telegram_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: +def test_telegram_performance_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) - patch_get_signal(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade + create_mock_trades_usdt(fee) - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False telegram._performance(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'Performance' in msg_mock.call_args_list[0][0][0] - assert 'ETH/BTC\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] + assert 'XRP/USDT\t9.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0] def test_telegram_entry_tag_performance_handle( - default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, mocker) -> None: + default_conf_usdt, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) patch_get_signal(freqtradebot) - # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade + create_mock_trades_usdt(fee) - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - trade.enter_tag = "TESTBUY" - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False context = MagicMock() telegram._enter_tag_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Entry Tag Performance' in msg_mock.call_args_list[0][0][0] - assert 'TESTBUY\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] + assert 'TEST1\t3.987 USDT (5.00%) (1)' in msg_mock.call_args_list[0][0][0] - context.args = [trade.pair] + context.args = ['XRP/USDT'] telegram._enter_tag_performance(update=update, context=context) assert msg_mock.call_count == 2 @@ -1312,37 +1285,24 @@ def test_telegram_entry_tag_performance_handle( assert "Error" in msg_mock.call_args_list[0][0][0] -def test_telegram_exit_reason_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: +def test_telegram_exit_reason_performance_handle(default_conf_usdt, update, ticker, fee, + mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) patch_get_signal(freqtradebot) - # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade - trade.exit_reason = 'TESTSELL' - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) + create_mock_trades_usdt(fee) - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False context = MagicMock() telegram._exit_reason_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Exit Reason Performance' in msg_mock.call_args_list[0][0][0] - assert 'TESTSELL\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] - context.args = [trade.pair] + assert 'roi\t9.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0] + context.args = ['XRP/USDT'] telegram._exit_reason_performance(update=update, context=context) assert msg_mock.call_count == 2 @@ -1356,43 +1316,27 @@ def test_telegram_exit_reason_performance_handle(default_conf, update, ticker, f assert "Error" in msg_mock.call_args_list[0][0][0] -def test_telegram_mix_tag_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, mocker) -> None: +def test_telegram_mix_tag_performance_handle(default_conf_usdt, update, ticker, fee, + mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) patch_get_signal(freqtradebot) # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade - - trade.enter_tag = "TESTBUY" - trade.exit_reason = "TESTSELL" - - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False + create_mock_trades_usdt(fee) context = MagicMock() telegram._mix_tag_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Mix Tag Performance' in msg_mock.call_args_list[0][0][0] - assert ('TESTBUY TESTSELL\t0.00006217 BTC (6.20%) (1)' + assert ('TEST3 roi\t9.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0]) - context.args = [trade.pair] + context.args = ['XRP/USDT'] telegram._mix_tag_performance(update=update, context=context) assert msg_mock.call_count == 2 From 40c7caac16279c9d1e34ef50fe2fc8178b01d886 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 03:01:53 +0000 Subject: [PATCH 079/162] Bump types-filelock from 3.2.6 to 3.2.7 Bumps [types-filelock](https://github.com/python/typeshed) from 3.2.6 to 3.2.7. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-filelock dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4eb157aae..e7d64a2b6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.0.1 -types-filelock==3.2.6 +types-filelock==3.2.7 types-requests==2.27.30 types-tabulate==0.8.9 types-python-dateutil==2.8.17 From 390e600f55cfe868bee74d9d74fc03e323575359 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 06:46:34 +0200 Subject: [PATCH 080/162] Update statistics output --- tests/conftest_trades_usdt.py | 104 +++++++++++++++++----------------- tests/rpc/test_rpc.py | 87 +++++++++------------------- 2 files changed, 78 insertions(+), 113 deletions(-) diff --git a/tests/conftest_trades_usdt.py b/tests/conftest_trades_usdt.py index cc1b1a206..41d705c01 100644 --- a/tests/conftest_trades_usdt.py +++ b/tests/conftest_trades_usdt.py @@ -20,36 +20,60 @@ def direc(is_short: bool): def mock_order_usdt_1(is_short: bool): return { - 'id': f'1234_{direc(is_short)}', - 'symbol': 'ADA/USDT', + 'id': f'prod_entry_1_{direc(is_short)}', + 'symbol': 'LTC/USDT', 'status': 'closed', 'side': entry_side(is_short), 'type': 'limit', - 'price': 2.0, - 'amount': 10.0, - 'filled': 10.0, + 'price': 10.0, + 'amount': 2.0, + 'filled': 2.0, + 'remaining': 0.0, + } + + +def mock_order_usdt_1_exit(is_short: bool): + return { + 'id': f'prod_exit_1_{direc(is_short)}', + 'symbol': 'LTC/USDT', + 'status': 'closed', + 'side': exit_side(is_short), + 'type': 'limit', + 'price': 8.0, + 'amount': 2.0, + 'filled': 2.0, 'remaining': 0.0, } def mock_trade_usdt_1(fee, is_short: bool): + """ + Simulate prod entry with open sell order + """ trade = Trade( - pair='ADA/USDT', + pair='LTC/USDT', stake_amount=20.0, - amount=10.0, - amount_requested=10.0, + amount=2.0, + amount_requested=2.0, + open_date=datetime.now(tz=timezone.utc) - timedelta(days=2, minutes=20), + close_date=datetime.now(tz=timezone.utc) - timedelta(days=2, minutes=5), fee_open=fee.return_value, fee_close=fee.return_value, - is_open=True, - open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), - open_rate=2.0, + is_open=False, + open_rate=10.0, + close_rate=8.0, + close_profit=-0.2, + close_profit_abs=-4.0, exchange='binance', - open_order_id=f'1234_{direc(is_short)}', - strategy='StrategyTestV2', + strategy='SampleStrategy', + open_order_id=f'prod_exit_1_{direc(is_short)}', timeframe=5, is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_1(is_short), 'ADA/USDT', entry_side(is_short)) + o = Order.parse_from_ccxt_object(mock_order_usdt_1(is_short), 'LTC/USDT', entry_side(is_short)) + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_usdt_1_exit(is_short), + 'LTC/USDT', exit_side(is_short)) trade.orders.append(o) return trade @@ -330,59 +354,35 @@ def mock_trade_usdt_6(fee, is_short: bool): def mock_order_usdt_7(is_short: bool): return { - 'id': f'prod_entry_7_{direc(is_short)}', - 'symbol': 'LTC/USDT', + 'id': f'1234_{direc(is_short)}', + 'symbol': 'ADA/USDT', 'status': 'closed', 'side': entry_side(is_short), 'type': 'limit', - 'price': 10.0, - 'amount': 2.0, - 'filled': 2.0, - 'remaining': 0.0, - } - - -def mock_order_usdt_7_exit(is_short: bool): - return { - 'id': f'prod_exit_7_{direc(is_short)}', - 'symbol': 'LTC/USDT', - 'status': 'closed', - 'side': exit_side(is_short), - 'type': 'limit', - 'price': 8.0, - 'amount': 2.0, - 'filled': 2.0, + 'price': 2.0, + 'amount': 10.0, + 'filled': 10.0, 'remaining': 0.0, } def mock_trade_usdt_7(fee, is_short: bool): - """ - Simulate prod entry with open sell order - """ trade = Trade( - pair='LTC/USDT', + pair='ADA/USDT', stake_amount=20.0, - amount=2.0, - amount_requested=2.0, - open_date=datetime.now(tz=timezone.utc) - timedelta(days=2, minutes=20), - close_date=datetime.now(tz=timezone.utc) - timedelta(days=2, minutes=5), + amount=10.0, + amount_requested=10.0, fee_open=fee.return_value, fee_close=fee.return_value, - is_open=False, - open_rate=10.0, - close_rate=8.0, - close_profit=-0.2, - close_profit_abs=-4.0, + is_open=True, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), + open_rate=2.0, exchange='binance', - strategy='SampleStrategy', - open_order_id=f'prod_exit_7_{direc(is_short)}', + open_order_id=f'1234_{direc(is_short)}', + strategy='StrategyTestV2', timeframe=5, is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_7(is_short), 'LTC/USDT', entry_side(is_short)) - trade.orders.append(o) - o = Order.parse_from_ccxt_object(mock_order_usdt_7_exit(is_short), - 'LTC/USDT', exit_side(is_short)) + o = Order.parse_from_ccxt_object(mock_order_usdt_7(is_short), 'ADA/USDT', entry_side(is_short)) trade.orders.append(o) return trade diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 0273b8237..339a6382f 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -407,13 +407,9 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog, is_short): assert stoploss_mock.call_count == 0 -def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, mocker) -> None: - mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', - get_price=MagicMock(return_value={'bitcoin': {'usd': 15000.0}}), - ) - mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) +def test_rpc_trade_statistics11(default_conf_usdt, ticker, fee, + mocker) -> None: + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=1.1) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -421,10 +417,9 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - patch_get_signal(freqtradebot) - stake_currency = default_conf['stake_currency'] - fiat_display_currency = default_conf['fiat_display_currency'] + freqtradebot = get_patched_freqtradebot(mocker, default_conf_usdt) + stake_currency = default_conf_usdt['stake_currency'] + fiat_display_currency = default_conf_usdt['fiat_display_currency'] rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() @@ -437,62 +432,32 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert res['latest_trade_timestamp'] == 0 # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'sell') - trade.update_trade(oobj) - - # Update the ticker with a market going up - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up - ) - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - trade.close_date = datetime.utcnow() - trade.is_open = False - - freqtradebot.enter_positions() - trade = Trade.query.first() - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Update the ticker with a market going up - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up - ) - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - trade.close_date = datetime.utcnow() - trade.is_open = False + create_mock_trades_usdt(fee) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) - assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) - assert prec_satoshi(stats['profit_closed_percent_mean'], 6.2) - assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) - assert prec_satoshi(stats['profit_all_coin'], 5.802e-05) - assert prec_satoshi(stats['profit_all_percent_mean'], 2.89) - assert prec_satoshi(stats['profit_all_fiat'], 0.8703) - assert stats['trade_count'] == 2 - assert stats['first_trade_date'] == 'just now' - assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] in ('0:00:00', '0:00:01', '0:00:02') - assert stats['best_pair'] == 'ETH/BTC' - assert prec_satoshi(stats['best_rate'], 6.2) + assert pytest.approx(stats['profit_closed_coin']) == 9.83 + assert pytest.approx(stats['profit_closed_percent_mean']) == -1.67 + assert pytest.approx(stats['profit_closed_fiat']) == 10.813 + assert pytest.approx(stats['profit_all_coin']) == -77.45964918 + assert pytest.approx(stats['profit_all_percent_mean']) == -57.86 + assert pytest.approx(stats['profit_all_fiat']) == -85.205614098 + assert stats['trade_count'] == 7 + assert stats['first_trade_date'] == '2 days ago' + assert stats['latest_trade_date'] == '17 minutes ago' + assert stats['avg_duration'] in ('0:17:40') + assert stats['best_pair'] == 'XRP/USDT' + assert stats['best_rate'] == 10.0 # Test non-available pair mocker.patch('freqtrade.exchange.Exchange.get_rate', - MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) + MagicMock(side_effect=ExchangeError("Pair 'XRP/USDT' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) - assert stats['trade_count'] == 2 - assert stats['first_trade_date'] == 'just now' - assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] in ('0:00:00', '0:00:01', '0:00:02') - assert stats['best_pair'] == 'ETH/BTC' - assert prec_satoshi(stats['best_rate'], 6.2) + assert stats['trade_count'] == 7 + assert stats['first_trade_date'] == '2 days ago' + assert stats['latest_trade_date'] == '17 minutes ago' + assert stats['avg_duration'] in ('0:17:40') + assert stats['best_pair'] == 'XRP/USDT' + assert stats['best_rate'] == 10.0 assert isnan(stats['profit_all_coin']) From 43c871f2f4e4b7022befea6e4dd8c3b8871231a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 06:49:31 +0200 Subject: [PATCH 081/162] Use time-machine to stabilize time-sensitive tests --- tests/rpc/test_rpc_telegram.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 48acda47e..3bd817ac7 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -405,7 +405,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: assert msg_mock.call_count == 1 -def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: +def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker, time_machine) -> None: mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=1.1 @@ -418,6 +418,8 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) + # Move date to within day + time_machine.move_to('2022-06-11 08:00:00+00:00') # Create some test data create_mock_trades_usdt(fee) @@ -491,7 +493,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: assert 'Daily Profit over the last 7 days:' in msg_mock.call_args_list[0][0][0] -def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: +def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker, time_machine) -> None: default_conf_usdt['max_open_trades'] = 1 mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', @@ -504,7 +506,8 @@ def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: ) telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) - + # Move to saturday - so all trades are within that week + time_machine.move_to('2022-06-11') create_mock_trades_usdt(fee) # Try valid data @@ -560,7 +563,7 @@ def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: ) -def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: +def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker, time_machine) -> None: default_conf_usdt['max_open_trades'] = 1 mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', @@ -573,7 +576,8 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker) -> None: ) telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) - + # Move to day within the month so all mock trades fall into this week. + time_machine.move_to('2022-06-11') create_mock_trades_usdt(fee) # Try valid data From 8fd245c28b6d41be9b45a8c9f5aeb6d5ab7d277c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 06:58:06 +0200 Subject: [PATCH 082/162] Update pre-commit filelocktypes --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 685d789ec..f5c1a36f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: exclude: build_helpers additional_dependencies: - types-cachetools==5.0.1 - - types-filelock==3.2.6 + - types-filelock==3.2.7 - types-requests==2.27.30 - types-tabulate==0.8.9 - types-python-dateutil==2.8.17 From 70966c8a8f0782b1e4d3f94c64b8cecb0e34b71b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 05:08:12 +0000 Subject: [PATCH 083/162] Bump actions/setup-python from 3 to 4 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 3 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbe0bcf6e..551268af7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -127,7 +127,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -211,7 +211,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -263,7 +263,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" @@ -282,7 +282,7 @@ jobs: ./tests/test_docs.sh - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" @@ -336,7 +336,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.9" From e67d29cd2f85ac2eb029b1c7904ece2cc7cc35a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 07:10:47 +0200 Subject: [PATCH 084/162] Update more trades to use create_mock_trades --- tests/rpc/test_rpc.py | 179 +++++++++++------------------------------- 1 file changed, 46 insertions(+), 133 deletions(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 339a6382f..d20646e60 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -11,7 +11,6 @@ from freqtrade.edge import PairInfo from freqtrade.enums import SignalDirection, State, TradingMode from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError from freqtrade.persistence import Trade -from freqtrade.persistence.models import Order from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter @@ -407,8 +406,7 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog, is_short): assert stoploss_mock.call_count == 0 -def test_rpc_trade_statistics11(default_conf_usdt, ticker, fee, - mocker) -> None: +def test_rpc_trade_statistics(default_conf_usdt, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=1.1) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -463,14 +461,9 @@ def test_rpc_trade_statistics11(default_conf_usdt, ticker, fee, # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) -def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, - ticker_sell_up, limit_buy_order, limit_sell_order): - mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', - get_price=MagicMock(return_value={'bitcoin': {'usd': 15000.0}}), - ) +def test_rpc_trade_statistics_closed(mocker, default_conf_usdt, ticker, fee): mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', - return_value=15000.0) + return_value=1.1) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -478,46 +471,32 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(freqtradebot) - stake_currency = default_conf['stake_currency'] - fiat_display_currency = default_conf['fiat_display_currency'] + stake_currency = default_conf_usdt['stake_currency'] + fiat_display_currency = default_conf_usdt['fiat_display_currency'] rpc = RPC(freqtradebot) # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - # Update the ticker with a market going up - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up, - get_fee=fee - ) - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - trade.close_date = datetime.utcnow() - trade.is_open = False + create_mock_trades_usdt(fee) for trade in Trade.query.order_by(Trade.id).all(): trade.open_rate = None stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) - assert prec_satoshi(stats['profit_closed_coin'], 0) - assert prec_satoshi(stats['profit_closed_percent_mean'], 0) - assert prec_satoshi(stats['profit_closed_fiat'], 0) - assert prec_satoshi(stats['profit_all_coin'], 0) - assert prec_satoshi(stats['profit_all_percent_mean'], 0) - assert prec_satoshi(stats['profit_all_fiat'], 0) - assert stats['trade_count'] == 1 - assert stats['first_trade_date'] == 'just now' - assert stats['latest_trade_date'] == 'just now' + assert stats['profit_closed_coin'] == 0 + assert stats['profit_closed_percent_mean'] == 0 + assert stats['profit_closed_fiat'] == 0 + assert stats['profit_all_coin'] == 0 + assert stats['profit_all_percent_mean'] == 0 + assert stats['profit_all_fiat'] == 0 + assert stats['trade_count'] == 7 + assert stats['first_trade_date'] == '2 days ago' + assert stats['latest_trade_date'] == '17 minutes ago' assert stats['avg_duration'] == '0:00:00' - assert stats['best_pair'] == 'ETH/BTC' - assert prec_satoshi(stats['best_rate'], 6.2) + assert stats['best_pair'] == 'XRP/USDT' + assert stats['best_rate'] == 10.0 def test_rpc_balance_handle_error(default_conf, mocker): @@ -869,8 +848,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 3 -def test_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: +def test_performance_handle(default_conf_usdt, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -879,34 +857,21 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(freqtradebot) rpc = RPC(freqtradebot) - # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade + create_mock_trades_usdt(fee) - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False res = rpc._rpc_performance() - assert len(res) == 1 - assert res[0]['pair'] == 'ETH/BTC' + assert len(res) == 3 + assert res[0]['pair'] == 'XRP/USDT' assert res[0]['count'] == 1 - assert prec_satoshi(res[0]['profit_pct'], 6.2) + assert res[0]['profit_pct'] == 10.0 -def test_enter_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: +def test_enter_tag_performance_handle(default_conf, ticker, fee, mocker) -> None: + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -920,34 +885,22 @@ def test_enter_tag_performance_handle(default_conf, ticker, limit_buy_order, fee rpc = RPC(freqtradebot) # Create some test data + create_mock_trades_usdt(fee) freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False res = rpc._rpc_enter_tag_performance(None) - assert len(res) == 1 - assert res[0]['enter_tag'] == 'Other' + assert len(res) == 3 + assert res[0]['enter_tag'] == 'TEST3' assert res[0]['count'] == 1 - assert prec_satoshi(res[0]['profit_pct'], 6.2) + assert res[0]['profit_pct'] == 10.0 - trade.enter_tag = "TEST_TAG" res = rpc._rpc_enter_tag_performance(None) - assert len(res) == 1 - assert res[0]['enter_tag'] == 'TEST_TAG' + assert len(res) == 3 + assert res[0]['enter_tag'] == 'TEST3' assert res[0]['count'] == 1 - assert prec_satoshi(res[0]['profit_pct'], 6.2) + assert res[0]['profit_pct'] == 10.0 def test_enter_tag_performance_handle_2(mocker, default_conf, markets, fee): @@ -979,8 +932,7 @@ def test_enter_tag_performance_handle_2(mocker, default_conf, markets, fee): assert prec_satoshi(res[0]['profit_pct'], 0.5) -def test_exit_reason_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: +def test_exit_reason_performance_handle(default_conf_usdt, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -989,39 +941,22 @@ def test_exit_reason_performance_handle(default_conf, ticker, limit_buy_order, f get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(freqtradebot) rpc = RPC(freqtradebot) # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade + create_mock_trades_usdt(fee) - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False res = rpc._rpc_exit_reason_performance(None) - assert len(res) == 1 - assert res[0]['exit_reason'] == 'Other' + assert len(res) == 3 + assert res[0]['exit_reason'] == 'roi' assert res[0]['count'] == 1 - assert prec_satoshi(res[0]['profit_pct'], 6.2) + assert res[0]['profit_pct'] == 10.0 - trade.exit_reason = "TEST1" - res = rpc._rpc_exit_reason_performance(None) - - assert len(res) == 1 - assert res[0]['exit_reason'] == 'TEST1' - assert res[0]['count'] == 1 - assert prec_satoshi(res[0]['profit_pct'], 6.2) + assert res[1]['exit_reason'] == 'exit_signal' + assert res[2]['exit_reason'] == 'Other' def test_exit_reason_performance_handle_2(mocker, default_conf, markets, fee): @@ -1053,8 +988,7 @@ def test_exit_reason_performance_handle_2(mocker, default_conf, markets, fee): assert prec_satoshi(res[0]['profit_pct'], 0.5) -def test_mix_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, mocker) -> None: +def test_mix_tag_performance_handle(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1068,35 +1002,14 @@ def test_mix_tag_performance_handle(default_conf, ticker, limit_buy_order, fee, rpc = RPC(freqtradebot) # Create some test data - freqtradebot.enter_positions() - trade = Trade.query.first() - assert trade + create_mock_trades_usdt(fee) - # Simulate fulfilled LIMIT_BUY order for trade - oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy') - trade.update_trade(oobj) - - # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell') - trade.update_trade(oobj) - - trade.close_date = datetime.utcnow() - trade.is_open = False res = rpc._rpc_mix_tag_performance(None) - assert len(res) == 1 - assert res[0]['mix_tag'] == 'Other Other' + assert len(res) == 3 + assert res[0]['mix_tag'] == 'TEST3 roi' assert res[0]['count'] == 1 - assert prec_satoshi(res[0]['profit_pct'], 6.2) - - trade.enter_tag = "TESTBUY" - trade.exit_reason = "TESTSELL" - res = rpc._rpc_mix_tag_performance(None) - - assert len(res) == 1 - assert res[0]['mix_tag'] == 'TESTBUY TESTSELL' - assert res[0]['count'] == 1 - assert prec_satoshi(res[0]['profit_pct'], 6.2) + assert res[0]['profit_pct'] == 10.0 def test_mix_tag_performance_handle_2(mocker, default_conf, markets, fee): From ee0b9e3a5c3aa907d8901db89e5c375ccb42b406 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 06:25:18 +0000 Subject: [PATCH 085/162] Bump mkdocs-material from 8.3.2 to 8.3.4 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.3.2 to 8.3.4. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.3.2...8.3.4) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index f351151ab..1b4403b97 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.3.2 +mkdocs-material==8.3.4 mdx_truly_sane_lists==1.2 pymdown-extensions==9.4 jinja2==3.1.2 From 71f314d4c45b39a91380c6b1a02876506b6430af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 06:25:35 +0000 Subject: [PATCH 086/162] Bump ccxt from 1.85.57 to 1.87.12 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.85.57 to 1.87.12. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.85.57...1.87.12) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 05d5a10db..4ebcdaa8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.85.57 +ccxt==1.87.12 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 43b8b0a083d527318f6033b26a395d8c5dbc7e86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 06:25:53 +0000 Subject: [PATCH 087/162] Bump mypy from 0.960 to 0.961 Bumps [mypy](https://github.com/python/mypy) from 0.960 to 0.961. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.960...v0.961) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e7d64a2b6..19912d59c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ coveralls==3.3.1 flake8==4.0.1 flake8-tidy-imports==4.8.0 -mypy==0.960 +mypy==0.961 pre-commit==2.19.0 pytest==7.1.2 pytest-asyncio==0.18.3 From cb2f89bca63a73aea5b35f5a6b8d2ae48d5455c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 06:26:23 +0000 Subject: [PATCH 088/162] Bump requests from 2.27.1 to 2.28.0 Bumps [requests](https://github.com/psf/requests) from 2.27.1 to 2.28.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.27.1...v2.28.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 05d5a10db..ba9cecafd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ SQLAlchemy==1.4.37 python-telegram-bot==13.12 arrow==1.2.2 cachetools==4.2.2 -requests==2.27.1 +requests==2.28.0 urllib3==1.26.9 jsonschema==4.6.0 TA-Lib==0.4.24 From fdca583c6760a6ba76f04b076e373d09accf8291 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 07:07:39 +0000 Subject: [PATCH 089/162] Bump pymdown-extensions from 9.4 to 9.5 Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 9.4 to 9.5. - [Release notes](https://github.com/facelessuser/pymdown-extensions/releases) - [Commits](https://github.com/facelessuser/pymdown-extensions/compare/9.4...9.5) --- updated-dependencies: - dependency-name: pymdown-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 1b4403b97..1f342ca02 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 mkdocs-material==8.3.4 mdx_truly_sane_lists==1.2 -pymdown-extensions==9.4 +pymdown-extensions==9.5 jinja2==3.1.2 From 850f5d3842008406c9a24611fdb6e40e6c138ae1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 07:32:39 +0000 Subject: [PATCH 090/162] Bump orjson from 3.7.1 to 3.7.2 Bumps [orjson](https://github.com/ijl/orjson) from 3.7.1 to 3.7.2. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.7.1...3.7.2) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9b7d87e02..b2dbd921e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.6 # Properly format api responses -orjson==3.7.1 +orjson==3.7.2 # Notify systemd sdnotify==0.3.2 From 35adeb64122a02d52f2515b53ea3bd125c2a8d31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 07:33:30 +0000 Subject: [PATCH 091/162] Bump plotly from 5.8.0 to 5.8.2 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.8.0 to 5.8.2. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.8.0...v5.8.2) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index e17efbc71..a2a894c57 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.8.0 +plotly==5.8.2 From 848a5d85c63f7655f958c700e1e82caaa69d2b9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 08:50:12 +0000 Subject: [PATCH 092/162] Add small stability fix to test --- tests/test_freqtradebot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 7f9bc6a46..3fd16f925 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3775,6 +3775,7 @@ def test_exit_profit_only( trade = Trade.query.first() assert trade.is_short == is_short oobj = Order.parse_from_ccxt_object(limit_order[eside], limit_order[eside]['symbol'], eside) + trade.update_order(limit_order[eside]) trade.update_trade(oobj) freqtrade.wallets.update() if profit_only: @@ -4063,6 +4064,7 @@ def test_trailing_stop_loss_positive( trade = Trade.query.first() assert trade.is_short == is_short oobj = Order.parse_from_ccxt_object(limit_order[eside], limit_order[eside]['symbol'], eside) + trade.update_order(limit_order[eside]) trade.update_trade(oobj) caplog.set_level(logging.DEBUG) # stop-loss not reached From d5fd1f9c3848469b4ce1fa1039ef533acc09c0f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 13:24:27 +0000 Subject: [PATCH 093/162] Improve order filled handling --- freqtrade/persistence/trade_model.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 0be9d22c1..79f58591d 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -74,7 +74,7 @@ class Order(_DECL_BASE): @property def safe_filled(self) -> float: - return self.filled or self.amount or 0.0 + return self.filled if self.filled is not None else self.amount or 0.0 @property def safe_fee_base(self) -> float: @@ -847,8 +847,6 @@ class LocalTrade(): tmp_amount = o.safe_amount_after_fee tmp_price = o.average or o.price - if o.filled is not None: - tmp_amount = o.filled if tmp_amount > 0.0 and tmp_price is not None: total_amount += tmp_amount total_stake += tmp_price * tmp_amount From 1ffee96bade938c25d025bd32839fb0a8a6430c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 19:59:00 +0200 Subject: [PATCH 094/162] Fix protection parameters not loading from parameter file closes #6978 --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index fdccc2f8a..000f74a27 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -73,8 +73,6 @@ class FreqtradeBot(LoggingMixin): PairLocks.timeframe = self.config['timeframe'] - self.protections = ProtectionManager(self.config, self.strategy.protections) - # RPC runs in separate threads, can start handling external commands just after # initialization, even before Freqtradebot has a chance to start its throttling, # so anything in the Freqtradebot instance should be ready (initialized), including @@ -124,6 +122,8 @@ class FreqtradeBot(LoggingMixin): self.last_process = datetime(1970, 1, 1, tzinfo=timezone.utc) self.strategy.ft_bot_start() + # Initialize protections AFTER bot start - otherwise parameters are not loaded. + self.protections = ProtectionManager(self.config, self.strategy.protections) def notify_status(self, msg: str) -> None: """ From 01a68e1060bf0c7a30a8fc9732d035547f7dd11c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Jun 2022 20:48:49 +0200 Subject: [PATCH 095/162] Remove unnecessary check and condition --- freqtrade/persistence/trade_model.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 79f58591d..3222a57b8 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -828,14 +828,6 @@ class LocalTrade(): return float(f"{profit_ratio:.8f}") def recalc_trade_from_orders(self): - # We need at least 2 entry orders for averaging amounts and rates. - # TODO: this condition could probably be removed - if len(self.select_filled_orders(self.entry_side)) < 2: - self.stake_amount = self.amount * self.open_rate / self.leverage - - # Just in case, still recalc open trade value - self.recalc_open_trade_value() - return total_amount = 0.0 total_stake = 0.0 From 6bb342f23a28559eca7c2355edd6e4af73b7e112 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 14 Jun 2022 16:54:27 +0100 Subject: [PATCH 096/162] Add export-filename support --- docs/advanced-backtesting.md | 25 +++++++++++++++++++++++++ freqtrade/commands/analyze_commands.py | 26 ++++++++++++++++---------- freqtrade/commands/arguments.py | 2 +- freqtrade/data/entryexitanalysis.py | 2 +- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 7f2be1f1a..457c487e9 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -45,6 +45,31 @@ ranging from the simplest (0) to the most detailed per pair, per buy and per sel More options are available by running with the `-h` option. +### Using export-filename + +Normally, `backtesting-analysis` uses the latest backtest results, but if you wanted to go +back to a previous backtest output, you need to supply the `--export-filename` option. +You can supply the same parameter to `backtest-analysis` with the name of the final backtest +output file. This allows you to keep historical versions of backtest results and reanalyse +them at a later date: + +``` bash +freqtrade backtesting -c --timeframe --strategy --timerange= --export=signals --export-filename=/tmp/mystrat_backtest.json +``` + +You should see some output similar to below in the logs with the name of the timestamped +filename that was exported: + +``` +2022-06-14 16:28:32,698 - freqtrade.misc - INFO - dumping json to "/tmp/mystrat_backtest-2022-06-14_16-28-32.json" +``` + +You can then use that filename in `backtesting-analysis`: + +``` +freqtrade backtesting-analysis -c --export-filename=/tmp/mystrat_backtest-2022-06-14_16-28-32.json +``` + ### Tuning the buy tags and sell tags to display To show only certain buy and sell tags in the displayed output, use the following two options: diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 2fa13f683..b6b790788 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -25,17 +25,23 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s if method in no_unlimited_runmodes.keys(): from freqtrade.data.btanalysis import get_latest_backtest_filename - btfile = Path(get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results')) - signals_file = f"{btfile.stem}_signals.pkl" + if 'exportfilename' in config: + if config['exportfilename'].is_dir(): + btfile = Path(get_latest_backtest_filename(config['exportfilename'])) + signals_file = f"{config['exportfilename']}/{btfile.stem}_signals.pkl" + else: + if config['exportfilename'].exists(): + btfile = Path(config['exportfilename']) + signals_file = f"{btfile.parent}/{btfile.stem}_signals.pkl" + else: + raise OperationalException(f"{config['exportfilename']} does not exist.") + else: + raise OperationalException('exportfilename not in config.') - if (not (config['user_data_dir'] / 'backtest_results' / signals_file).exists()): + if (not Path(signals_file).exists()): raise OperationalException( - "Cannot find latest backtest signals file. Run backtesting with --export signals." - ) - - if ('strategy' not in config): - raise OperationalException( - "No strategy defined. Use --strategy or supply in config." + (f"Cannot find latest backtest signals file: {signals_file}." + "Run backtesting with `--export signals`.") ) return config @@ -54,7 +60,7 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: logger.info('Starting freqtrade in analysis mode') - process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), + process_entry_exit_reasons(config['exportfilename'], config['exchange']['pair_whitelist'], config['analysis_groups'], config['enter_reason_list'], diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 6092c630b..1e3e2845a 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,7 +101,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] -ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", +ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason_list", "exit_reason_list", "indicator_list"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 1c21fcc15..d67064bd7 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -20,7 +20,7 @@ def _load_signal_candles(backtest_dir: Path): Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl" ) else: - scpf = Path(Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl") + scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_signals.pkl") try: scp = open(scpf, "rb") From 3c62df6b86c4749991ea751e0b567ea3df7585ac Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 06:53:52 +0200 Subject: [PATCH 097/162] Ensure the same timestamp is used for backtest and signal export --- freqtrade/optimize/backtesting.py | 7 ++++--- freqtrade/optimize/optimize_reports.py | 20 ++++++++++---------- freqtrade/rpc/api_server/api_backtest.py | 6 +++++- tests/optimize/test_optimize_reports.py | 14 +++++++------- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1aad8520a..77eb12419 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1264,13 +1264,14 @@ class Backtesting: self.results['strategy_comparison'].extend(results['strategy_comparison']) else: self.results = results - + dt_appendix = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") if self.config.get('export', 'none') in ('trades', 'signals'): - store_backtest_stats(self.config['exportfilename'], self.results) + store_backtest_stats(self.config['exportfilename'], self.results, dt_appendix) if (self.config.get('export', 'none') == 'signals' and self.dataprovider.runmode == RunMode.BACKTEST): - store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) + store_backtest_signal_candles( + self.config['exportfilename'], self.processed_dfs, dt_appendix) # Results may be mixed up now. Sort them so they follow --strategy-list order. if 'strategy_list' in self.config and len(self.results) > 0: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index e3dd17411..44b524a4c 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -17,21 +17,21 @@ from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) -def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> None: +def store_backtest_stats( + recordfilename: Path, stats: Dict[str, DataFrame], dtappendix: str) -> None: """ Stores backtest results :param recordfilename: Path object, which can either be a filename or a directory. Filenames will be appended with a timestamp right before the suffix while for directories, /backtest-result-.json will be used as filename :param stats: Dataframe containing the backtesting statistics + :param dtappendix: Datetime to use for the filename """ if recordfilename.is_dir(): - filename = (recordfilename / - f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json') + filename = (recordfilename / f'backtest-result-{dtappendix}.json') else: filename = Path.joinpath( - recordfilename.parent, - f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' + recordfilename.parent, f'{recordfilename.stem}-{dtappendix}' ).with_suffix(recordfilename.suffix) # Store metadata separately. @@ -44,7 +44,8 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) -def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> Path: +def store_backtest_signal_candles( + recordfilename: Path, candles: Dict[str, Dict], dtappendix: str) -> Path: """ Stores backtest trade signal candles :param recordfilename: Path object, which can either be a filename or a directory. @@ -52,14 +53,13 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict] while for directories, /backtest-result-_signals.pkl will be used as filename :param stats: Dict containing the backtesting signal candles + :param dtappendix: Datetime to use for the filename """ if recordfilename.is_dir(): - filename = (recordfilename / - f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl') + filename = (recordfilename / f'backtest-result-{dtappendix}_signals.pkl') else: filename = Path.joinpath( - recordfilename.parent, - f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl' + recordfilename.parent, f'{recordfilename.stem}-{dtappendix}_signals.pkl' ) file_dump_joblib(filename, candles) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 26b100408..06f04729b 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -1,6 +1,7 @@ import asyncio import logging from copy import deepcopy +from datetime import datetime from typing import Any, Dict, List from fastapi import APIRouter, BackgroundTasks, Depends @@ -102,7 +103,10 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac min_date=min_date, max_date=max_date) if btconfig.get('export', 'none') == 'trades': - store_backtest_stats(btconfig['exportfilename'], ApiServer._bt.results) + store_backtest_stats( + btconfig['exportfilename'], ApiServer._bt.results, + datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ) logger.info("Backtest finished.") diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 997c0436e..562e12820 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -171,7 +171,7 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): _backup_file(filename_last, copy_file=True) assert not filename.is_file() - store_backtest_stats(filename, stats) + store_backtest_stats(filename, stats, '2022_01_01_15_05_13') # get real Filename (it's btresult-.json) last_fn = get_latest_backtest_filename(filename_last.parent) @@ -194,7 +194,7 @@ def test_store_backtest_stats(testdatadir, mocker): dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_json') - store_backtest_stats(testdatadir, {'metadata': {}}) + store_backtest_stats(testdatadir, {'metadata': {}}, '2022_01_01_15_05_13') assert dump_mock.call_count == 3 assert isinstance(dump_mock.call_args_list[0][0][0], Path) @@ -202,7 +202,7 @@ def test_store_backtest_stats(testdatadir, mocker): dump_mock.reset_mock() filename = testdatadir / 'testresult.json' - store_backtest_stats(filename, {'metadata': {}}) + store_backtest_stats(filename, {'metadata': {}}, '2022_01_01_15_05_13') assert dump_mock.call_count == 3 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-.json @@ -216,7 +216,7 @@ def test_store_backtest_candles(testdatadir, mocker): candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} # mock directory exporting - store_backtest_signal_candles(testdatadir, candle_dict) + store_backtest_signal_candles(testdatadir, candle_dict, '2022_01_01_15_05_13') assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) @@ -225,7 +225,7 @@ def test_store_backtest_candles(testdatadir, mocker): dump_mock.reset_mock() # mock file exporting filename = Path(testdatadir / 'testresult') - store_backtest_signal_candles(filename, candle_dict) + store_backtest_signal_candles(filename, candle_dict, '2022_01_01_15_05_13') assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-_signals.pkl @@ -238,7 +238,7 @@ def test_write_read_backtest_candles(tmpdir): candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} # test directory exporting - stored_file = store_backtest_signal_candles(Path(tmpdir), candle_dict) + stored_file = store_backtest_signal_candles(Path(tmpdir), candle_dict, '2022_01_01_15_05_13') scp = open(stored_file, "rb") pickled_signal_candles = joblib.load(scp) scp.close() @@ -252,7 +252,7 @@ def test_write_read_backtest_candles(tmpdir): # test file exporting filename = Path(tmpdir / 'testresult') - stored_file = store_backtest_signal_candles(filename, candle_dict) + stored_file = store_backtest_signal_candles(filename, candle_dict, '2022_01_01_15_05_13') scp = open(stored_file, "rb") pickled_signal_candles = joblib.load(scp) scp.close() From 29d8aeb9b3c19c5b5c7a37d8d8dc42e6478f6f33 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 07:13:47 +0200 Subject: [PATCH 098/162] Don't fail on invalid parameter --- freqtrade/data/entryexitanalysis.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index d67064bd7..999f27955 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -140,14 +140,16 @@ def _do_group_table_output(bigdf, glist): # 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large) if g == "4": group_mask = ['pair', 'enter_reason', 'exit_reason'] + if group_mask: + new = bigdf.groupby(group_mask).agg(agg_mask).reset_index() + new.columns = group_mask + agg_cols + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 - new = bigdf.groupby(group_mask).agg(agg_mask).reset_index() - new.columns = group_mask + agg_cols - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 - - _print_table(new, sortcols) + _print_table(new, sortcols) + else: + logger.warning("Invalid group mask specified.") def _print_results(analysed_trades, stratname, analysis_groups, From c391ca08ded88ff1c4edfd8ab9a40b385b0a16a2 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 15 Jun 2022 11:25:06 +0100 Subject: [PATCH 099/162] Change backtesting-analysis options to space separated lists --- docs/advanced-backtesting.md | 13 +++++----- freqtrade/commands/cli_options.py | 17 +++++++------ freqtrade/data/entryexitanalysis.py | 38 +++++++++++++---------------- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 457c487e9..b6b75c47d 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -28,10 +28,11 @@ backtesting with the `--cache none` option to make sure no cached results are us If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the `user_data/backtest_results` folder. -To analyze the entry/exit tags, we now need to use the `freqtrade backtesting-analysis` command: +To analyze the entry/exit tags, we now need to use the `freqtrade backtesting-analysis` command +with `--analysis-groups` option provided with space-separated arguments (default `0 1 2`): ``` bash -freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 +freqtrade backtesting-analysis -c --analysis-groups 0 1 2 3 4 ``` This command will read from the last backtesting results. The `--analysis-groups` option is @@ -75,14 +76,14 @@ freqtrade backtesting-analysis -c --export-filename=/tmp/mystrat_b To show only certain buy and sell tags in the displayed output, use the following two options: ``` ---enter-reason-list : Comma separated list of enter signals to analyse. Default: "all" ---exit-reason-list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss" +--enter-reason-list : Space-separated list of enter signals to analyse. Default: "all" +--exit-reason-list : Space-separated list of exit signals to analyse. Default: "all" ``` For example: ```bash -freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" +freqtrade backtesting-analysis -c --analysis-groups 0 2 --enter-reason-list enter_tag_a enter_tag_b --exit-reason-list roi custom_exit_tag_a stop_loss ``` ### Outputting signal candle indicators @@ -93,7 +94,7 @@ indicators. To print out a column for a given set of indicators, use the `--indi option: ```bash -freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" --indicator-list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +freqtrade backtesting-analysis -c --analysis-groups 0 2 --enter-reason-list enter_tag_a enter_tag_b --exit-reason-list roi custom_exit_tag_a stop_loss --indicator-list rsi rsi_1h bb_lowerband ema_9 macd macdsignal ``` The indicators have to be present in your strategy's main DataFrame (either for your main diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index e90d3478d..3370ce64b 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -622,28 +622,29 @@ AVAILABLE_CLI_OPTIONS = { "2: by enter_tag and exit_tag, " "3: by pair and enter_tag, " "4: by pair, enter_ and exit_tag (this can get quite large)"), - nargs='?', - default="0,1,2", + nargs='+', + default=['0', '1', '2'], + choices=['0', '1', '2', '3', '4'], ), "enter_reason_list": Arg( "--enter-reason-list", help=("Comma separated list of entry signals to analyse. Default: all. " "e.g. 'entry_tag_a,entry_tag_b'"), - nargs='?', - default='all', + nargs='+', + default=['all'], ), "exit_reason_list": Arg( "--exit-reason-list", help=("Comma separated list of exit signals to analyse. Default: all. " "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), - nargs='?', - default='all', + nargs='+', + default=['all'], ), "indicator_list": Arg( "--indicator-list", help=("Comma separated list of indicators to analyse. " "e.g. 'close,rsi,bb_lowerband,profit_abs'"), - nargs='?', - default='', + nargs='+', + default=[], ), } diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index d67064bd7..6a157debb 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -161,28 +161,24 @@ def _print_results(analysed_trades, stratname, analysis_groups, bigdf = pd.concat([bigdf, trades], ignore_index=True) if bigdf.shape[0] > 0 and ('enter_reason' in bigdf.columns): - if analysis_groups is not None: - glist = analysis_groups.split(",") - _do_group_table_output(bigdf, glist) + if analysis_groups: + _do_group_table_output(bigdf, analysis_groups) - if enter_reason_list is not None and not enter_reason_list == "all": - enter_reason_list = enter_reason_list.split(",") + if enter_reason_list and "all" not in enter_reason_list: bigdf = bigdf.loc[(bigdf['enter_reason'].isin(enter_reason_list))] - if exit_reason_list is not None and not exit_reason_list == "all": - exit_reason_list = exit_reason_list.split(",") + if exit_reason_list and "all" not in exit_reason_list: bigdf = bigdf.loc[(bigdf['exit_reason'].isin(exit_reason_list))] - if indicator_list is not None and indicator_list != "": - if indicator_list == "all": - print(bigdf) - else: - available_inds = [] - for ind in indicator_list.split(","): - if ind in bigdf: - available_inds.append(ind) - ilist = ["pair", "enter_reason", "exit_reason"] + available_inds - _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) + if "all" in indicator_list: + print(bigdf) + elif indicator_list is not None: + available_inds = [] + for ind in indicator_list: + if ind in bigdf: + available_inds.append(ind) + ilist = ["pair", "enter_reason", "exit_reason"] + available_inds + _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) else: print("\\_ No trades to show") @@ -205,10 +201,10 @@ def _print_table(df, sortcols=None, show_index=False): def process_entry_exit_reasons(backtest_dir: Path, pairlist: List[str], - analysis_groups: Optional[str] = "0,1,2", - enter_reason_list: Optional[str] = "all", - exit_reason_list: Optional[str] = "all", - indicator_list: Optional[str] = None): + analysis_groups: Optional[List[str]] = ["0", "1", "2"], + enter_reason_list: Optional[List[str]] = ["all"], + exit_reason_list: Optional[List[str]] = ["all"], + indicator_list: Optional[List[str]] = []): try: backtest_stats = load_backtest_stats(backtest_dir) for strategy_name, results in backtest_stats['strategy'].items(): From 4a5ed5a2731bc78522c2ed3118398f64f538975e Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 15 Jun 2022 11:48:57 +0100 Subject: [PATCH 100/162] Fix tests --- tests/data/test_entryexitanalysis.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 6209110fe..09fbe9957 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -103,8 +103,8 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp # test group 0 and indicator list args = get_args(base_args + - ['--analysis-groups', '0', - '--indicator-list', 'close,rsi,profit_abs'] + ['--analysis-groups', "0", + '--indicator-list', "close", "rsi", "profit_abs"] ) start_analysis_entries_exits(args) captured = capsys.readouterr() @@ -128,7 +128,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '47.0996' in captured.out # test group 1 - args = get_args(base_args + ['--analysis-groups', '1']) + args = get_args(base_args + ['--analysis-groups', "1"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -141,7 +141,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '0' in captured.out # test group 2 - args = get_args(base_args + ['--analysis-groups', '2']) + args = get_args(base_args + ['--analysis-groups', "2"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -156,7 +156,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2.5' in captured.out # test group 3 - args = get_args(base_args + ['--analysis-groups', '3']) + args = get_args(base_args + ['--analysis-groups', "3"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out @@ -171,7 +171,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2' in captured.out # test group 4 - args = get_args(base_args + ['--analysis-groups', '4']) + args = get_args(base_args + ['--analysis-groups', "4"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out From e2e6c790be9869e8bb82d9bf58f8a2d8f98c0cec Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 16:50:25 +0200 Subject: [PATCH 101/162] Minor doc update --- docs/advanced-backtesting.md | 2 +- docs/utils.md | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index b6b75c47d..5c2500f18 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -51,7 +51,7 @@ More options are available by running with the `-h` option. Normally, `backtesting-analysis` uses the latest backtest results, but if you wanted to go back to a previous backtest output, you need to supply the `--export-filename` option. You can supply the same parameter to `backtest-analysis` with the name of the final backtest -output file. This allows you to keep historical versions of backtest results and reanalyse +output file. This allows you to keep historical versions of backtest results and re-analyse them at a later date: ``` bash diff --git a/docs/utils.md b/docs/utils.md index f87aa2ffc..0dd88b242 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -660,26 +660,31 @@ More details in the [Backtesting analysis](advanced-backtesting.md#analyze-the-b ``` usage: freqtrade backtesting-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] - [--analysis-groups [ANALYSIS_GROUPS]] - [--enter-reason-list [ENTER_REASON_LIST]] - [--exit-reason-list [EXIT_REASON_LIST]] - [--indicator-list [INDICATOR_LIST]] + [--export-filename PATH] + [--analysis-groups {0,1,2,3,4} [{0,1,2,3,4} ...]] + [--enter-reason-list ENTER_REASON_LIST [ENTER_REASON_LIST ...]] + [--exit-reason-list EXIT_REASON_LIST [EXIT_REASON_LIST ...]] + [--indicator-list INDICATOR_LIST [INDICATOR_LIST ...]] optional arguments: -h, --help show this help message and exit - --analysis-groups [ANALYSIS_GROUPS] + --export-filename PATH, --backtest-filename PATH + Use this filename for backtest results.Requires + `--export` to be set as well. Example: `--export-filen + ame=user_data/backtest_results/backtest_today.json` + --analysis-groups {0,1,2,3,4} [{0,1,2,3,4} ...] grouping output - 0: simple wins/losses by enter tag, 1: by enter_tag, 2: by enter_tag and exit_tag, 3: by pair and enter_tag, 4: by pair, enter_ and exit_tag (this can get quite large) - --enter-reason-list [ENTER_REASON_LIST] + --enter-reason-list ENTER_REASON_LIST [ENTER_REASON_LIST ...] Comma separated list of entry signals to analyse. Default: all. e.g. 'entry_tag_a,entry_tag_b' - --exit-reason-list [EXIT_REASON_LIST] + --exit-reason-list EXIT_REASON_LIST [EXIT_REASON_LIST ...] Comma separated list of exit signals to analyse. Default: all. e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss' - --indicator-list [INDICATOR_LIST] + --indicator-list INDICATOR_LIST [INDICATOR_LIST ...] Comma separated list of indicators to analyse. e.g. 'close,rsi,bb_lowerband,profit_abs' From f9e2e87346319bb32e596538a91340b8b4474b09 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 20:03:36 +0200 Subject: [PATCH 102/162] Improve some formatting and typehints --- freqtrade/rpc/rpc.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index da5144dab..8b1cdb851 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -512,7 +512,7 @@ class RPC: def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: """ Returns current account balance per crypto """ - currencies = [] + currencies: List[Dict] = [] total = 0.0 try: tickers = self._freqtrade.exchange.get_tickers(cached=True) @@ -547,13 +547,12 @@ class RPC: except (ExchangeError): logger.warning(f" Could not get rate for pair {coin}.") continue - total = total + (est_stake or 0) + total = total + est_stake currencies.append({ 'currency': coin, - # TODO: The below can be simplified if we don't assign None to values. - 'free': balance.free if balance.free is not None else 0, - 'balance': balance.total if balance.total is not None else 0, - 'used': balance.used if balance.used is not None else 0, + 'free': balance.free, + 'balance': balance.total, + 'used': balance.used, 'est_stake': est_stake or 0, 'stake': stake_currency, 'side': 'long', @@ -583,7 +582,6 @@ class RPC: total, stake_currency, fiat_display_currency) if self._fiat_converter else 0 trade_count = len(Trade.get_trades_proxy()) - starting_capital_ratio = 0.0 starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0 starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0 @@ -871,7 +869,7 @@ class RPC: else: errors[pair] = { 'error_msg': f"Pair {pair} is not in the current blacklist." - } + } resp = self._rpc_blacklist() resp['errors'] = errors return resp From 8f32fa5cb30efed0fa4f3d81e676951b5554dd8a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 20:13:07 +0200 Subject: [PATCH 103/162] Avoid exception on exchange recycling if __init__ fails --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c1a9059a7..465cce300 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -93,7 +93,7 @@ class Exchange: :return: None """ self._api: ccxt.Exchange - self._api_async: ccxt_async.Exchange + self._api_async: ccxt_async.Exchange = None self._markets: Dict = {} self._trading_fees: Dict[str, Any] = {} self._leverage_tiers: Dict[str, List[Dict]] = {} From 14a859c190a1fa8b89a13b8d756127546b822ff5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Jun 2022 19:50:13 +0200 Subject: [PATCH 104/162] Improve some documentation around futures / leverage --- docs/leverage.md | 15 ++++++++++++++- docs/stoploss.md | 13 +++++++++++++ docs/strategy-callbacks.md | 3 +++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/leverage.md b/docs/leverage.md index 2ee6f8444..491e6eda0 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -64,7 +64,10 @@ You will also have to pick a "margin mode" (explanation below) - with freqtrade ### Margin mode -The possible values are: `isolated`, or `cross`(*currently unavailable*) +On top of `trading_mode` - you will also have to configure your `margin_mode`. +While freqtrade currently only supports one margin mode, this will change, and by configuring it now you're all set for future updates. + +The possible values are: `isolated`, or `cross`(*currently unavailable*). #### Isolated margin mode @@ -82,6 +85,16 @@ One account is used to share collateral between markets (trading pairs). Margin "margin_mode": "cross" ``` +## Set leverage to use + +Different strategies and risk profiles will require different levels of leverage. +While you could configure one static leverage value - freqtrade offers you the flexibility to adjust this via [strategy leverage callback](strategy-callbacks.md#leverage-callback) - which allows you to use different leverages by pair, or based on some other factor benefitting your strategy result. + +If not implemented, leverage defaults to 1x (no leverage). + +!!! Warning + Higher leverage also equals higher risk - be sure you fully understand the implications of using leverage! + ## Understand `liquidation_buffer` *Defaults to `0.05`* diff --git a/docs/stoploss.md b/docs/stoploss.md index 573fdbd6c..83f787947 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -191,6 +191,19 @@ For example, simplified math: !!! Tip Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade. +## Stoploss and Leverage + +Stoploss should be thought of as "risk on this trade" - so a stoploss of 10% on a 100$ trade means you are willing to lose 10$ (10%) on this trade - which would trigger if the price moves 10% to the downside. + +When using leverage, the same principle is applied - with stoploss defining the risk on the trade (the amount you are willing to lose). + +Therefore, a stoploss of 10% on a 10x trade would trigger on a 1% price move. +If your stake amount (own capital) was 100$ - this trade would be 1000$ at 10x (after leverage). +If price moves 1% - you've lost 10$ of your own capital - therfore stoploss will trigger in this case. + +Make sure to be aware of this, and avoid using too tight stoploss (at 10x leverage, 10% risk may be too little to allow the trade to "breath" a little). + + ## Changing stoploss on open trades A stoploss on an open trade can be changed by changing the value in the configuration or strategy and use the `/reload_config` command (alternatively, completely stopping and restarting the bot also works). diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 410641f44..beffba56b 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -823,3 +823,6 @@ class AwesomeStrategy(IStrategy): """ return 1.0 ``` + +All profit calculations include leverage. Stoploss / ROI also include leverage in their calculation. +Defining a stoploss of 10% at 10x leverage would trigger the stoploss with a 1% move to the downside. From 575b4ead1a04bffc4c3064abcfe85fe77574b7c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 06:29:17 +0000 Subject: [PATCH 105/162] Update Test with funding_fee 0 --- tests/test_persistence.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index be19a3f5f..836b17a55 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -884,6 +884,17 @@ def test_calc_close_trade_price( ('binance', False, 3, 2.2, 0.0025, 4.684999, 0.23366583, futures, -1), ('binance', True, 1, 2.2, 0.0025, -7.315, -0.12222222, futures, -1), ('binance', True, 3, 2.2, 0.0025, -7.315, -0.36666666, futures, -1), + + # FUTURES, funding_fee=0 + ('binance', False, 1, 2.1, 0.0025, 2.6925, 0.04476309, futures, 0), + ('binance', False, 3, 2.1, 0.0025, 2.6925, 0.13428928, futures, 0), + ('binance', True, 1, 2.1, 0.0025, -3.3074999, -0.05526316, futures, 0), + ('binance', True, 3, 2.1, 0.0025, -3.3074999, -0.16578947, futures, 0), + + ('binance', False, 1, 1.9, 0.0025, -3.2925, -0.05473815, futures, 0), + ('binance', False, 3, 1.9, 0.0025, -3.2925, -0.16421446, futures, 0), + ('binance', True, 1, 1.9, 0.0025, 2.7075, 0.0452381, futures, 0), + ('binance', True, 3, 1.9, 0.0025, 2.7075, 0.13571429, futures, 0), ]) @pytest.mark.usefixtures("init_persistence") def test_calc_profit( From 76cae8e8e3b2710983084acbfa6a0e88fa0e2c81 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 06:53:40 +0000 Subject: [PATCH 106/162] Update tests to always provide rate to profit calculations --- tests/rpc/test_rpc_apiserver.py | 8 ++++---- tests/test_freqtradebot.py | 2 +- tests/test_persistence.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8b3ac18ac..ada1a82ec 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -852,8 +852,8 @@ def test_api_performance(botclient, fee): close_rate=0.265441, ) - trade.close_profit = trade.calc_profit_ratio() - trade.close_profit_abs = trade.calc_profit() + trade.close_profit = trade.calc_profit_ratio(trade.close_rate) + trade.close_profit_abs = trade.calc_profit(trade.close_rate) Trade.query.session.add(trade) trade = Trade( @@ -868,8 +868,8 @@ def test_api_performance(botclient, fee): fee_open=fee.return_value, close_rate=0.391 ) - trade.close_profit = trade.calc_profit_ratio() - trade.close_profit_abs = trade.calc_profit() + trade.close_profit = trade.calc_profit_ratio(trade.close_rate) + trade.close_profit_abs = trade.calc_profit(trade.close_rate) Trade.query.session.add(trade) Trade.commit() diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3fd16f925..4f3d5f667 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2151,7 +2151,7 @@ def test_handle_trade( assert trade.close_rate == 2.0 if is_short else 2.2 assert trade.close_profit == close_profit - assert trade.calc_profit() == 5.685 + assert trade.calc_profit(trade.close_rate) == 5.685 assert trade.close_date is not None assert trade.exit_reason == 'sell_signal1' diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 836b17a55..de250e3e6 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -606,9 +606,9 @@ def test_calc_open_close_trade_price( trade.close_rate = 2.2 trade.recalc_open_trade_value() assert isclose(trade._calc_open_trade_value(), open_value) - assert isclose(trade.calc_close_trade_value(), close_value) - assert isclose(trade.calc_profit(), round(profit, 8)) - assert pytest.approx(trade.calc_profit_ratio()) == profit_ratio + assert isclose(trade.calc_close_trade_value(trade.close_rate), close_value) + assert isclose(trade.calc_profit(trade.close_rate), round(profit, 8)) + assert pytest.approx(trade.calc_profit_ratio(trade.close_rate)) == profit_ratio @pytest.mark.usefixtures("init_persistence") @@ -660,7 +660,7 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee): trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') trade.update_trade(oobj) - assert trade.calc_close_trade_value() == 0.0 + assert trade.calc_close_trade_value(trade.close_rate) == 0.0 @pytest.mark.usefixtures("init_persistence") From d7770c507b4e655a74226eada048d86fe7d6fdb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 07:00:42 +0000 Subject: [PATCH 107/162] Remove implicit use of certain rates in profit calculations --- freqtrade/persistence/trade_model.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3222a57b8..5a89849dd 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -624,8 +624,8 @@ class LocalTrade(): """ self.close_rate = rate self.close_date = self.close_date or datetime.utcnow() - self.close_profit = self.calc_profit_ratio() - self.close_profit_abs = self.calc_profit() + self.close_profit = self.calc_profit_ratio(rate) + self.close_profit_abs = self.calc_profit(rate) self.is_open = False self.exit_order_status = 'closed' self.open_order_id = None @@ -714,10 +714,10 @@ class LocalTrade(): return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) - def _calc_base_close(self, amount: Decimal, rate: Optional[float] = None, + def _calc_base_close(self, amount: Decimal, rate: float, fee: Optional[float] = None) -> Decimal: - close_trade = Decimal(amount) * Decimal(rate or self.close_rate) # type: ignore + close_trade = Decimal(amount) * Decimal(rate) fees = close_trade * Decimal(fee or self.fee_close) if self.is_short: @@ -725,15 +725,14 @@ class LocalTrade(): else: return close_trade - fees - def calc_close_trade_value(self, rate: Optional[float] = None, + def calc_close_trade_value(self, rate: float, fee: Optional[float] = None, interest_rate: Optional[float] = None) -> float: """ Calculate the close_rate including fee + :param rate: rate to compare with. :param fee: fee to use on the close rate (optional). If rate is not set self.fee will be used - :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used :param interest_rate: interest_charge for borrowing this coin (optional). If interest_rate is not set self.interest_rate will be used :return: Price in BTC of the open trade @@ -770,21 +769,20 @@ class LocalTrade(): raise OperationalException( f"{self.trading_mode.value} trading is not yet available using freqtrade") - def calc_profit(self, rate: Optional[float] = None, + def calc_profit(self, rate: float, fee: Optional[float] = None, interest_rate: Optional[float] = None) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade + :param rate: close rate to compare with. :param fee: fee to use on the close rate (optional). If fee is not set self.fee will be used - :param rate: close rate to compare with (optional). - If rate is not set self.close_rate will be used :param interest_rate: interest_charge for borrowing this coin (optional). If interest_rate is not set self.interest_rate will be used :return: profit in stake currency as float """ close_trade_value = self.calc_close_trade_value( - rate=(rate or self.close_rate), + rate=rate, fee=(fee or self.fee_close), interest_rate=(interest_rate or self.interest_rate) ) @@ -795,20 +793,19 @@ class LocalTrade(): profit = close_trade_value - self.open_trade_value return float(f"{profit:.8f}") - def calc_profit_ratio(self, rate: Optional[float] = None, + def calc_profit_ratio(self, rate: float, fee: Optional[float] = None, interest_rate: Optional[float] = None) -> float: """ Calculates the profit as ratio (including fee). - :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used + :param rate: rate to compare with. :param fee: fee to use on the close rate (optional). :param interest_rate: interest_charge for borrowing this coin (optional). If interest_rate is not set self.interest_rate will be used :return: profit ratio as float """ close_trade_value = self.calc_close_trade_value( - rate=(rate or self.close_rate), + rate=rate, fee=(fee or self.fee_close), interest_rate=(interest_rate or self.interest_rate) ) From 91f9818ae3b1c95ca2547916d7ea70be6c2a84a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 09:53:29 +0000 Subject: [PATCH 108/162] Simplify trade calculations --- freqtrade/persistence/trade_model.py | 43 +++++++++------------------- tests/test_persistence.py | 2 +- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 5a89849dd..3ac64ba6b 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -693,10 +693,9 @@ class LocalTrade(): """ self.open_trade_value = self._calc_open_trade_value() - def calculate_interest(self, interest_rate: Optional[float] = None) -> Decimal: + def calculate_interest(self) -> Decimal: """ - :param interest_rate: interest_charge for borrowing this coin(optional). - If interest_rate is not set self.interest_rate will be used + Calculate interest for this trade. Only applicable for Margin trading. """ zero = Decimal(0.0) # If nothing was borrowed @@ -709,7 +708,7 @@ class LocalTrade(): total_seconds = Decimal((now - open_date).total_seconds()) hours = total_seconds / sec_per_hour or zero - rate = Decimal(interest_rate or self.interest_rate) + rate = Decimal(self.interest_rate) borrowed = Decimal(self.borrowed) return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) @@ -726,16 +725,13 @@ class LocalTrade(): return close_trade - fees def calc_close_trade_value(self, rate: float, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: + fee: Optional[float] = None) -> float: """ - Calculate the close_rate including fee + Calculate the Trade's close value including fees :param rate: rate to compare with. :param fee: fee to use on the close rate (optional). - If rate is not set self.fee will be used - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used - :return: Price in BTC of the open trade + If rate is not set self.close_fee will be used + :return: value in stake currency of the open trade """ if rate is None and not self.close_rate: return 0.0 @@ -748,7 +744,7 @@ class LocalTrade(): elif (trading_mode == TradingMode.MARGIN): - total_interest = self.calculate_interest(interest_rate) + total_interest = self.calculate_interest() if self.is_short: amount = amount + total_interest @@ -769,22 +765,15 @@ class LocalTrade(): raise OperationalException( f"{self.trading_mode.value} trading is not yet available using freqtrade") - def calc_profit(self, rate: float, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: + def calc_profit(self, rate: float) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade :param rate: close rate to compare with. - :param fee: fee to use on the close rate (optional). - If fee is not set self.fee will be used - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used - :return: profit in stake currency as float + :return: profit in stake currency as float """ close_trade_value = self.calc_close_trade_value( rate=rate, - fee=(fee or self.fee_close), - interest_rate=(interest_rate or self.interest_rate) + fee=self.fee_close ) if self.is_short: @@ -793,21 +782,15 @@ class LocalTrade(): profit = close_trade_value - self.open_trade_value return float(f"{profit:.8f}") - def calc_profit_ratio(self, rate: float, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: + def calc_profit_ratio(self, rate: float) -> float: """ Calculates the profit as ratio (including fee). :param rate: rate to compare with. - :param fee: fee to use on the close rate (optional). - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used :return: profit ratio as float """ close_trade_value = self.calc_close_trade_value( rate=rate, - fee=(fee or self.fee_close), - interest_rate=(interest_rate or self.interest_rate) + fee=self.fee_close ) short_close_zero = (self.is_short and close_trade_value == 0.0) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index de250e3e6..8c12d2ea0 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -813,7 +813,7 @@ def test_calc_close_trade_price( funding_fees=funding_fees ) trade.open_order_id = 'close_trade' - assert round(trade.calc_close_trade_value(rate=close_rate, fee=fee_rate), 8) == result + assert round(trade.calc_close_trade_value(rate=close_rate), 8) == result @pytest.mark.parametrize( From 6bdf9c2a94a53820dc658d6b006f194e16b6e7f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 11:17:05 +0000 Subject: [PATCH 109/162] Simplify trade profit calculations further --- freqtrade/persistence/trade_model.py | 32 ++++++++++------------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3ac64ba6b..eb405942a 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -713,24 +713,20 @@ class LocalTrade(): return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) - def _calc_base_close(self, amount: Decimal, rate: float, - fee: Optional[float] = None) -> Decimal: + def _calc_base_close(self, amount: Decimal, rate: float, fee: float) -> Decimal: - close_trade = Decimal(amount) * Decimal(rate) - fees = close_trade * Decimal(fee or self.fee_close) + close_trade = amount * Decimal(rate) + fees = close_trade * Decimal(fee) if self.is_short: return close_trade + fees else: return close_trade - fees - def calc_close_trade_value(self, rate: float, - fee: Optional[float] = None) -> float: + def calc_close_trade_value(self, rate: float) -> float: """ Calculate the Trade's close value including fees :param rate: rate to compare with. - :param fee: fee to use on the close rate (optional). - If rate is not set self.close_fee will be used :return: value in stake currency of the open trade """ if rate is None and not self.close_rate: @@ -740,7 +736,7 @@ class LocalTrade(): trading_mode = self.trading_mode or TradingMode.SPOT if trading_mode == TradingMode.SPOT: - return float(self._calc_base_close(amount, rate, fee)) + return float(self._calc_base_close(amount, rate, self.fee_close)) elif (trading_mode == TradingMode.MARGIN): @@ -748,19 +744,19 @@ class LocalTrade(): if self.is_short: amount = amount + total_interest - return float(self._calc_base_close(amount, rate, fee)) + return float(self._calc_base_close(amount, rate, self.fee_close)) else: # Currency already owned for longs, no need to purchase - return float(self._calc_base_close(amount, rate, fee) - total_interest) + return float(self._calc_base_close(amount, rate, self.fee_close) - total_interest) elif (trading_mode == TradingMode.FUTURES): funding_fees = self.funding_fees or 0.0 # Positive funding_fees -> Trade has gained from fees. # Negative funding_fees -> Trade had to pay the fees. if self.is_short: - return float(self._calc_base_close(amount, rate, fee)) - funding_fees + return float(self._calc_base_close(amount, rate, self.fee_close)) - funding_fees else: - return float(self._calc_base_close(amount, rate, fee)) + funding_fees + return float(self._calc_base_close(amount, rate, self.fee_close)) + funding_fees else: raise OperationalException( f"{self.trading_mode.value} trading is not yet available using freqtrade") @@ -771,10 +767,7 @@ class LocalTrade(): :param rate: close rate to compare with. :return: profit in stake currency as float """ - close_trade_value = self.calc_close_trade_value( - rate=rate, - fee=self.fee_close - ) + close_trade_value = self.calc_close_trade_value(rate) if self.is_short: profit = self.open_trade_value - close_trade_value @@ -788,10 +781,7 @@ class LocalTrade(): :param rate: rate to compare with. :return: profit ratio as float """ - close_trade_value = self.calc_close_trade_value( - rate=rate, - fee=self.fee_close - ) + close_trade_value = self.calc_close_trade_value(rate) short_close_zero = (self.is_short and close_trade_value == 0.0) long_close_zero = (not self.is_short and self.open_trade_value == 0.0) From fda8248d41bce429415cb20292550df6d7e5e654 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 22:43:24 +0200 Subject: [PATCH 110/162] Gateio allow market orders on futures markets --- freqtrade/exchange/gateio.py | 1 - tests/exchange/test_gateio.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index 4147e8290..f69a0dc02 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -40,7 +40,6 @@ class Gateio(Exchange): ] def validate_ordertypes(self, order_types: Dict) -> None: - super().validate_ordertypes(order_types) if self.trading_mode != TradingMode.FUTURES: if any(v == 'market' for k, v in order_types.items()): diff --git a/tests/exchange/test_gateio.py b/tests/exchange/test_gateio.py index 92f8186a6..cbd4776fb 100644 --- a/tests/exchange/test_gateio.py +++ b/tests/exchange/test_gateio.py @@ -33,6 +33,12 @@ def test_validate_order_types_gateio(default_conf, mocker): match=r'Exchange .* does not support market orders.'): ExchangeResolver.load_exchange('gateio', default_conf, True) + # market-orders supported on futures markets. + default_conf['trading_mode'] = 'futures' + default_conf['margin_mode'] = 'isolated' + ex = ExchangeResolver.load_exchange('gateio', default_conf, True) + assert ex + @pytest.mark.usefixtures("init_persistence") def test_fetch_stoploss_order_gateio(default_conf, mocker): From 616bf315cbf8647f5f9114aee74e648f747b9a27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Jun 2022 23:02:39 +0200 Subject: [PATCH 111/162] gateio: futures market orders require IOC to be set. --- freqtrade/exchange/gateio.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index f69a0dc02..fd9a2b2b3 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -3,6 +3,7 @@ import logging from datetime import datetime from typing import Dict, List, Optional, Tuple +from freqtrade.constants import BuySell from freqtrade.enums import MarginMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange @@ -24,6 +25,8 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, "ohlcv_volume_currency": "quote", + "time_in_force_parameter": "timeInForce", + "order_time_in_force": ['gtc', 'ioc'], "stoploss_order_types": {"limit": "limit"}, "stoploss_on_exchange": True, } @@ -46,6 +49,27 @@ class Gateio(Exchange): raise OperationalException( f'Exchange {self.name} does not support market orders.') + def _get_params( + self, + side: BuySell, + ordertype: str, + leverage: float, + reduceOnly: bool, + time_in_force: str = 'gtc', + ) -> Dict: + params = super()._get_params( + side=side, + ordertype=ordertype, + leverage=leverage, + reduceOnly=reduceOnly, + time_in_force=time_in_force, + ) + if ordertype == 'market' and self.trading_mode == TradingMode.FUTURES: + params['type'] = 'market' + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: 'ioc'}) + return params + def get_trades_for_order(self, order_id: str, pair: str, since: datetime, params: Optional[Dict] = None) -> List: trades = super().get_trades_for_order(order_id, pair, since, params) @@ -60,7 +84,8 @@ class Gateio(Exchange): pair_fees = self._trading_fees.get(pair, {}) if pair_fees: for idx, trade in enumerate(trades): - if trade.get('fee', {}).get('cost') is None: + fee = trade.get('fee', {}) + if fee and fee.get('cost') is None: takerOrMaker = trade.get('takerOrMaker', 'taker') if pair_fees.get(takerOrMaker) is not None: trades[idx]['fee'] = { From 017fd03180e09549c0fc4604e19581bf54a66de1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:05:22 +0200 Subject: [PATCH 112/162] Fix but with late entries in backtesting --- freqtrade/data/history/history_utils.py | 2 +- freqtrade/optimize/backtesting.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index bead59814..c972c841c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -221,7 +221,7 @@ def _download_pair_history(pair: str, *, prepend=prepend) logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, ' - f'{candle_type} and store in {datadir}.' + f'{candle_type} and store in {datadir}. ' f'From {format_ms_time(since_ms) if since_ms else "start"} to ' f'{format_ms_time(until_ms) if until_ms else "now"}' ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 77eb12419..f7d92081f 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1078,6 +1078,8 @@ class Backtesting: open_trade_count += 1 # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.") open_trades[pair].append(trade) + LocalTrade.add_bt_trade(trade) + self.wallets.update() for trade in list(open_trades[pair]): # 3. Process entry orders. @@ -1085,7 +1087,6 @@ class Backtesting: if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time, trade) trade.open_order_id = None - LocalTrade.add_bt_trade(trade) self.wallets.update() # 4. Create exit orders (if any) From d62273294d4cb2ba23e94f733a08dfa02000dee3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:10:33 +0200 Subject: [PATCH 113/162] Update /help for /fx to align with actual command name closes #6985 --- docs/telegram-usage.md | 5 +++-- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 6e21d3689..773a1b67a 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -171,8 +171,8 @@ official commands. You can ask at any moment for help with `/help`. | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). | `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) -| `/forceexit ` | Instantly exits the given trade (Ignoring `minimum_roi`). -| `/forceexit all` | Instantly exits all open trades (Ignoring `minimum_roi`). +| `/forceexit | /fx ` | Instantly exits the given trade (Ignoring `minimum_roi`). +| `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`). | `/fx` | alias for `/forceexit` | `/forcelong [rate]` | Instantly buys the given pair. Rate is optional and only applies to limit orders. (`force_entry_enable` must be set to True) | `/forceshort [rate]` | Instantly shorts the given pair. Rate is optional and only applies to limit orders. This will only work on non-spot markets. (`force_entry_enable` must be set to True) @@ -281,6 +281,7 @@ Starting capital is either taken from the `available_capital` setting, or calcul !!! Tip You can get a list of all open trades by calling `/forceexit` without parameter, which will show a list of buttons to simply exit a trade. + This command has an alias in `/fx` - which has the same capabilities, but is faster to type in "emergency" situations. ### /forcelong [rate] | /forceshort [rate] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2e1d23621..7f5da8872 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1395,7 +1395,7 @@ class Telegram(RPCHandler): "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" - "*/fe |all:* `Alias to /forceexit`\n" + "*/fx |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" "*/whitelist:* `Show current whitelist` \n" From 03815cb81bde3f568f3319760c577b8229878eb0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:23:16 +0200 Subject: [PATCH 114/162] Use fstrings in telegram messaging --- freqtrade/rpc/telegram.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7f5da8872..c595018d4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -326,33 +326,33 @@ class Telegram(RPCHandler): elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' - message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling {message_side} Order for {pair} (#{trade_id}). " - "Reason: {reason}.".format(**msg)) + message = (f"\N{WARNING SIGN} *{msg['exchange']}:* " + f"Cancelling {msg['message_side']} Order for {msg['pair']} " + f"(#{msg['trade_id']}). Reason: {msg['reason']}.") elif msg_type == RPCMessageType.PROTECTION_TRIGGER: message = ( - "*Protection* triggered due to {reason}. " - "`{pair}` will be locked until `{lock_end_time}`." - ).format(**msg) + f"*Protection* triggered due to {msg['reason']}. " + f"`{msg['pair']}` will be locked until `{msg['lock_end_time']}`." + ) elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: message = ( - "*Protection* triggered due to {reason}. " - "*All pairs* will be locked until `{lock_end_time}`." - ).format(**msg) + f"*Protection* triggered due to {msg['reason']}. " + f"*All pairs* will be locked until `{msg['lock_end_time']}`." + ) elif msg_type == RPCMessageType.STATUS: - message = '*Status:* `{status}`'.format(**msg) + message = f"*Status:* `{msg['status']}`" elif msg_type == RPCMessageType.WARNING: - message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) + message = f"\N{WARNING SIGN} *Warning:* `{msg['status']}`" elif msg_type == RPCMessageType.STARTUP: - message = '{status}'.format(**msg) + message = f"{msg['status']}" else: - raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + raise NotImplementedError(f"Unknown message type: {msg_type}") return message def send_msg(self, msg: Dict[str, Any]) -> None: @@ -867,7 +867,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_start() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _stop(self, update: Update, context: CallbackContext) -> None: @@ -879,7 +879,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_stop() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _reload_config(self, update: Update, context: CallbackContext) -> None: @@ -891,7 +891,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_reload_config() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _stopbuy(self, update: Update, context: CallbackContext) -> None: @@ -903,7 +903,7 @@ class Telegram(RPCHandler): :return: None """ msg = self._rpc._rpc_stopbuy() - self._send_msg('Status: `{status}`'.format(**msg)) + self._send_msg(f"Status: `{msg['status']}`") @authorized_only def _force_exit(self, update: Update, context: CallbackContext) -> None: @@ -1065,9 +1065,9 @@ class Telegram(RPCHandler): trade_id = int(context.args[0]) msg = self._rpc._rpc_delete(trade_id) self._send_msg(( - '`{result_msg}`\n' + f"`{msg['result_msg']}`\n" 'Please make sure to take care of this asset on the exchange manually.' - ).format(**msg)) + )) except RPCException as e: self._send_msg(str(e)) From d77ce468ea8fd196b09b36aafc98dcde8bebbfe5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 09:35:19 +0200 Subject: [PATCH 115/162] Add "dry" hint to buy/sell messages part of #6962 --- freqtrade/rpc/telegram.py | 14 +++++++++++--- tests/rpc/test_rpc_telegram.py | 19 +++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c595018d4..15e919e30 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -235,6 +235,14 @@ class Telegram(RPCHandler): # This can take up to `timeout` from the call to `start_polling`. self._updater.stop() + def _exchange_from_msg(self, msg: Dict[str, Any]) -> str: + """ + Extracts the exchange name from the given message. + :param msg: The message to extract the exchange name from. + :return: The exchange name. + """ + return f"{msg['exchange']}{' (dry)' if self._config['dry_run'] else ''}" + def _format_entry_msg(self, msg: Dict[str, Any]) -> str: if self._rpc._fiat_converter: msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( @@ -247,7 +255,7 @@ class Telegram(RPCHandler): entry_side = ({'enter': 'Long', 'entered': 'Longed'} if msg['direction'] == 'Long' else {'enter': 'Short', 'entered': 'Shorted'}) message = ( - f"{emoji} *{msg['exchange']}:*" + f"{emoji} *{self._exchange_from_msg(msg)}:*" f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}" f" (#{msg['trade_id']})\n" ) @@ -296,7 +304,7 @@ class Telegram(RPCHandler): msg['profit_extra'] = '' is_fill = msg['type'] == RPCMessageType.EXIT_FILL message = ( - f"{msg['emoji']} *{msg['exchange']}:* " + f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* " f"{'Exited' if is_fill else 'Exiting'} {msg['pair']} (#{msg['trade_id']})\n" f"*{'Profit' if is_fill else 'Unrealized Profit'}:* " f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n" @@ -326,7 +334,7 @@ class Telegram(RPCHandler): elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' - message = (f"\N{WARNING SIGN} *{msg['exchange']}:* " + message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* " f"Cancelling {msg['message_side']} Order for {msg['pair']} " f"(#{msg['trade_id']}). Reason: {msg['reason']}.") diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3bd817ac7..d6845be57 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1680,7 +1680,7 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog, message_type, leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{LARGE BLUE CIRCLE} *Binance:* {enter} ETH/BTC (#1)\n' + f'\N{LARGE BLUE CIRCLE} *Binance (dry):* {enter} ETH/BTC (#1)\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' f'{leverage_text}' @@ -1720,7 +1720,7 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker, message_type, en 'pair': 'ETH/BTC', 'reason': CANCEL_REASON['TIMEOUT'] }) - assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Binance:* ' + assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Binance (dry):* ' 'Cancelling enter Order for ETH/BTC (#1). ' 'Reason: cancelled due to timeout.') @@ -1782,7 +1782,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{CHECK MARK} *Binance:* {entered}ed ETH/BTC (#1)\n' + f'\N{CHECK MARK} *Binance (dry):* {entered}ed ETH/BTC (#1)\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' f"{leverage_text}" @@ -1820,7 +1820,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' @@ -1854,7 +1854,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' @@ -1883,10 +1883,12 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: 'reason': 'Cancelled on exchange' }) assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Cancelling exit Order for KEY/ETH (#1).' + '\N{WARNING SIGN} *Binance (dry):* Cancelling exit Order for KEY/ETH (#1).' ' Reason: Cancelled on exchange.') msg_mock.reset_mock() + # Test with live mode (no dry appendix) + telegram._config['dry_run'] = False telegram.send_msg({ 'type': RPCMessageType.EXIT_CANCEL, 'trade_id': 1, @@ -1935,7 +1937,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exited KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n' '*Profit:* `-57.41%`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' @@ -1991,6 +1993,7 @@ def test_send_msg_unknown_type(default_conf, mocker) -> None: def test_send_msg_buy_notification_no_fiat( default_conf, mocker, message_type, enter, enter_signal, leverage) -> None: del default_conf['fiat_display_currency'] + default_conf['dry_run'] = False telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram.send_msg({ @@ -2060,7 +2063,7 @@ def test_send_msg_sell_notification_no_fiat( leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance:* Exiting KEY/ETH (#1)\n' + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' '*Unrealized Profit:* `-57.41%`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' From 6a15d36d14525ff20083a788848230ee3150fa45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:14:28 +0200 Subject: [PATCH 116/162] Add Drawdown and profit_factor to /profit #6816 --- freqtrade/optimize/optimize_reports.py | 2 ++ freqtrade/rpc/api_server/api_schemas.py | 3 +++ freqtrade/rpc/rpc.py | 23 +++++++++++++++++++++++ freqtrade/rpc/telegram.py | 11 ++++++++--- tests/rpc/test_rpc_apiserver.py | 15 ++++++++++++--- tests/rpc/test_rpc_telegram.py | 5 +++-- 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 44b524a4c..79cb8a2bd 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -497,8 +497,10 @@ def generate_strategy_stats(pairlist: List[str], (drawdown_abs, drawdown_start, drawdown_end, high_val, low_val, max_drawdown) = calculate_max_drawdown( results, value_col='profit_abs', starting_balance=start_balance) + # max_relative_drawdown = Underwater (_, _, _, _, _, max_relative_drawdown) = calculate_max_drawdown( results, value_col='profit_abs', starting_balance=start_balance, relative=True) + strat_stats.update({ 'max_drawdown': max_drawdown_legacy, # Deprecated - do not use 'max_drawdown_account': max_drawdown, diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 11fdc0121..fda2d7ea0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -104,6 +104,9 @@ class Profit(BaseModel): best_pair_profit_ratio: float winning_trades: int losing_trades: int + profit_factor: float + max_drawdown: float + max_drawdown_abs: float class SellReason(BaseModel): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8b1cdb851..bae90b3bc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -18,6 +18,7 @@ from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.data.history import load_data +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State, TradingMode) from freqtrade.exceptions import ExchangeError, PricingError @@ -415,6 +416,8 @@ class RPC: durations = [] winning_trades = 0 losing_trades = 0 + winning_profit = 0.0 + losing_profit = 0.0 for trade in trades: current_rate: float = 0.0 @@ -430,8 +433,10 @@ class RPC: profit_closed_ratio.append(profit_ratio) if trade.close_profit >= 0: winning_trades += 1 + winning_profit += trade.close_profit_abs else: losing_trades += 1 + losing_profit += trade.close_profit_abs else: # Get current rate try: @@ -470,6 +475,21 @@ class RPC: profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance + profit_factor = winning_profit / abs(losing_profit) if losing_profit else float('inf') + + trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT), + 'profit_abs': trade.close_profit_abs} + for trade in trades if not trade.is_open]) + max_drawdown_abs = 0.0 + max_drawdown = 0.0 + if len(trades_df) > 0: + try: + (max_drawdown_abs, _, _, _, _, max_drawdown) = calculate_max_drawdown( + trades_df, value_col='profit_abs', starting_balance=starting_balance) + except ValueError: + # ValueError if no losing trade. + pass + profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, stake_currency, @@ -508,6 +528,9 @@ class RPC: 'best_pair_profit_ratio': best_pair[1] if best_pair else 0, 'winning_trades': winning_trades, 'losing_trades': losing_trades, + 'profit_factor': profit_factor, + 'max_drawdown': max_drawdown, + 'max_drawdown_abs': max_drawdown_abs, } def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 15e919e30..a7130d691 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -730,12 +730,17 @@ class Telegram(RPCHandler): f"*Total Trade Count:* `{trade_count}`\n" f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " f"`{first_trade_date}`\n" - f"*Latest Trade opened:* `{latest_trade_date}\n`" + f"*Latest Trade opened:* `{latest_trade_date}`\n" f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" ) if stats['closed_trade_count'] > 0: - markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" - f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`") + markdown_msg += ( + f"\n*Avg. Duration:* `{avg_duration}`\n" + f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n" + f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" + f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " + f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" + ) self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", query=update.callback_query) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index ada1a82ec..afbc92c5d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -724,7 +724,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': -83.19455985, 'profit_closed_ratio_mean': -0.0075, 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, - 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2} + 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2, + 'profit_factor': 0.0, + } ), ( False, @@ -737,7 +739,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': 9.124559849999999, 'profit_closed_ratio_mean': 0.0075, 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, - 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0} + 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0, + 'profit_factor': None, + } ), ( None, @@ -750,7 +754,9 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_fiat': -67.02260985, 'profit_closed_ratio_mean': 0.0025, 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, - 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1} + 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1, + 'profit_factor': 0.02775724835771106, + } ) ]) def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected): @@ -803,6 +809,9 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'closed_trade_count': 2, 'winning_trades': expected['winning_trades'], 'losing_trades': expected['losing_trades'], + 'profit_factor': expected['profit_factor'], + 'max_drawdown': ANY, + 'max_drawdown_abs': ANY, } diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index d6845be57..65917a6e2 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -704,11 +704,12 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '∙ `6.253 USD`' in msg_mock.call_args_list[-1][0][0] assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] + assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] + assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) -def test_telegram_stats(default_conf, update, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, mocker, is_short) -> None: +def test_telegram_stats(default_conf, update, ticker, fee, mocker, is_short) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', From 40c9abc7e1b2120d80884f00f260ce635a52d74a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:40:32 +0200 Subject: [PATCH 117/162] Add trading volume to /profit output --- freqtrade/persistence/trade_model.py | 16 ++++++++++++++++ freqtrade/rpc/rpc.py | 2 ++ freqtrade/rpc/telegram.py | 1 + tests/rpc/test_rpc_telegram.py | 1 + 4 files changed, 20 insertions(+) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index eb405942a..3a52c0660 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1352,3 +1352,19 @@ class Trade(_DECL_BASE, LocalTrade): .group_by(Trade.pair) \ .order_by(desc('profit_sum')).first() return best_pair + + @staticmethod + def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float: + """ + Get Trade volume based on Orders + NOTE: Not supported in Backtesting. + :returns: Tuple containing (pair, profit_sum) + """ + trading_volume = Order.query.with_entities( + func.sum(Order.cost).label('volume') + ).filter( + (Order.order_filled_date >= start_date) + & (Order.status == 'closed') + ) \ + .scalar() + return trading_volume diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index bae90b3bc..31fe4c469 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -452,6 +452,7 @@ class RPC: profit_all_ratio.append(profit_ratio) best_pair = Trade.get_best_pair(start_date) + trading_volume = Trade.get_trading_volume(start_date) # Prepare data to display profit_closed_coin_sum = round(sum(profit_closed_coin), 8) @@ -531,6 +532,7 @@ class RPC: 'profit_factor': profit_factor, 'max_drawdown': max_drawdown, 'max_drawdown_abs': max_drawdown_abs, + 'trading_volume': trading_volume, } def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a7130d691..58bfc6bf7 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -737,6 +737,7 @@ class Telegram(RPCHandler): markdown_msg += ( f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n" + f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n" f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 65917a6e2..e36d98083 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -706,6 +706,7 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] + assert '*Trading volume:* `60 USDT`' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) From b7e4dea6c5d6004e51a823f4bacdb5ac200d3016 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 11:43:50 +0200 Subject: [PATCH 118/162] Document new Profit metrics --- docs/telegram-usage.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 773a1b67a..95e7eaa16 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -270,10 +270,15 @@ Return a summary of your profit/loss and performance. > **Latest Trade opened:** `2 minutes ago` > **Avg. Duration:** `2:33:45` > **Best Performing:** `PAY/BTC: 50.23%` +> **Trading volume:** `0.5 BTC` +> **Profit factor:** `1.04` +> **Max Drawdown:** `9.23% (0.01255 BTC)` The relative profit of `1.2%` is the average profit per trade. The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. +Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. +Max drawdown corresponds to the backtesting metric `Absolute Drawdown (Account)` - calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`. ### /forceexit From 8c46d19071d96dae2eca4800e982b0e668e461d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:27:54 +0200 Subject: [PATCH 119/162] Fix backtesting bug balance was never released on cancelled trades --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f7d92081f..6eeefbfac 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1055,6 +1055,7 @@ class Backtesting: # Close trade open_trade_count -= 1 open_trades[pair].remove(t) + LocalTrade.trades_open.remove(t) self.wallets.update() # 2. Process entries. From 53bfa7931d1e4ac02a76a5b9be17d4fd6c55725a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:32:22 +0200 Subject: [PATCH 120/162] Add rudimentary test for prior bug Test fails without the fix in 8c46d19071d96dae2eca4800e982b0e668e461d6 --- tests/optimize/test_backtest_detail.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 4b4c446e0..a18196507 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -7,6 +7,7 @@ import pytest from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType from freqtrade.optimize.backtesting import Backtesting +from freqtrade.persistence.trade_model import LocalTrade from tests.conftest import patch_exchange from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset, tests_timeframe) @@ -964,5 +965,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) assert res.open_date == _get_frame_time_from_offset(trade.open_tick) assert res.close_date == _get_frame_time_from_offset(trade.close_tick) assert res.is_short == trade.is_short + assert len(LocalTrade.trades) == len(data.trades) + assert len(LocalTrade.trades_open) == 0 backtesting.cleanup() del backtesting From 474e6705e622aa7372c7a9b203df8553dbdcded3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:27:43 +0200 Subject: [PATCH 121/162] Add Profit factor to backtesting --- docs/backtesting.md | 4 ++++ docs/telegram-usage.md | 6 +++--- freqtrade/optimize/optimize_reports.py | 6 ++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 76718d206..50fc96923 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -300,6 +300,7 @@ A backtesting result will look like that: | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | | CAGR % | 460.87% | +| Profit factor | 1.11 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -399,6 +400,7 @@ It contains some useful key metrics about performance of your strategy on backte | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | | CAGR % | 460.87% | +| Profit factor | 1.11 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -444,6 +446,8 @@ It contains some useful key metrics about performance of your strategy on backte - `Final balance`: Final balance - starting balance + absolute profit. - `Absolute profit`: Profit made in stake currency. - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. +- `CAGR %`: Compound annual growth rate. +- `Profit factor`: profit / loss. - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. - `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 95e7eaa16..2145797b4 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -275,9 +275,9 @@ Return a summary of your profit/loss and performance. > **Max Drawdown:** `9.23% (0.01255 BTC)` The relative profit of `1.2%` is the average profit per trade. -The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. -Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. -Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. +The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. +Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. +Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy. Max drawdown corresponds to the backtesting metric `Absolute Drawdown (Account)` - calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`. ### /forceexit diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 79cb8a2bd..44ac4a5b3 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -416,6 +416,9 @@ def generate_strategy_stats(pairlist: List[str], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + winning_profit = results.loc[results['profit_abs'] > 0, 'profit_abs'].sum() + losing_profit = results.loc[results['profit_abs'] < 0, 'profit_abs'].sum() + profit_factor = winning_profit / abs(losing_profit) if losing_profit else 0.0 backtest_days = (max_date - min_date).days or 1 strat_stats = { @@ -443,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str], 'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(), 'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(), 'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']), + 'profit_factor': profit_factor, 'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT), 'backtest_start_ts': int(min_date.timestamp() * 1000), 'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT), @@ -779,6 +783,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: strat_results['stake_currency'])), ('Total profit %', f"{strat_results['profit_total']:.2%}"), ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), + ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor' + in strat_results else 'N/A'), ('Trades per day', strat_results['trades_per_day']), ('Avg. daily profit %', f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), From 0168343b7656eb572c0bda50daa774e2f9549e5f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 16:53:25 +0200 Subject: [PATCH 122/162] Add trading-volume to api schema --- freqtrade/persistence/trade_model.py | 7 +++---- freqtrade/rpc/api_server/api_schemas.py | 1 + tests/conftest_trades.py | 14 ++++++++++++++ tests/rpc/test_rpc_apiserver.py | 7 ++++--- tests/test_persistence.py | 1 + 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3a52c0660..39ebd75b4 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1363,8 +1363,7 @@ class Trade(_DECL_BASE, LocalTrade): trading_volume = Order.query.with_entities( func.sum(Order.cost).label('volume') ).filter( - (Order.order_filled_date >= start_date) - & (Order.status == 'closed') - ) \ - .scalar() + Order.order_filled_date >= start_date, + Order.status == 'closed' + ).scalar() return trading_volume diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index fda2d7ea0..7566e2ac0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -107,6 +107,7 @@ class Profit(BaseModel): profit_factor: float max_drawdown: float max_drawdown_abs: float + trading_volume: Optional[float] class SellReason(BaseModel): diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 006eab98f..1a8cf3183 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -29,6 +29,7 @@ def mock_order_1(is_short: bool): 'average': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -65,6 +66,7 @@ def mock_order_2(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -79,6 +81,7 @@ def mock_order_2_sell(is_short: bool): 'price': 0.128, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -126,6 +129,7 @@ def mock_order_3(is_short: bool): 'price': 0.05, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -141,6 +145,7 @@ def mock_order_3_sell(is_short: bool): 'average': 0.06, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -186,6 +191,7 @@ def mock_order_4(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 0.0, + 'cost': 15.129, 'remaining': 123.0, } @@ -225,6 +231,7 @@ def mock_order_5(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -239,6 +246,7 @@ def mock_order_5_stoploss(is_short: bool): 'price': 0.123, 'amount': 123.0, 'filled': 0.0, + 'cost': 0.0, 'remaining': 123.0, } @@ -281,6 +289,7 @@ def mock_order_6(is_short: bool): 'price': 0.15, 'amount': 2.0, 'filled': 2.0, + 'cost': 0.3, 'remaining': 0.0, } @@ -295,6 +304,7 @@ def mock_order_6_sell(is_short: bool): 'price': 0.15 if is_short else 0.20, 'amount': 2.0, 'filled': 0.0, + 'cost': 0.0, 'remaining': 2.0, } @@ -337,6 +347,7 @@ def short_order(): 'price': 0.123, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.129, 'remaining': 0.0, } @@ -351,6 +362,7 @@ def exit_short_order(): 'price': 0.128, 'amount': 123.0, 'filled': 123.0, + 'cost': 15.744, 'remaining': 0.0, } @@ -424,6 +436,7 @@ def leverage_order(): 'amount': 123.0, 'filled': 123.0, 'remaining': 0.0, + 'cost': 15.129, 'leverage': 5.0 } @@ -439,6 +452,7 @@ def leverage_order_sell(): 'amount': 123.0, 'filled': 123.0, 'remaining': 0.0, + 'cost': 15.744, 'leverage': 5.0 } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index afbc92c5d..b0ff5e1b2 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -725,7 +725,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2, - 'profit_factor': 0.0, + 'profit_factor': 0.0, 'trading_volume': 91.074, } ), ( @@ -740,7 +740,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0, - 'profit_factor': None, + 'profit_factor': None, 'trading_volume': 91.074, } ), ( @@ -755,7 +755,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1, - 'profit_factor': 0.02775724835771106, + 'profit_factor': 0.02775724835771106, 'trading_volume': 91.074, } ) ]) @@ -812,6 +812,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'profit_factor': expected['profit_factor'], 'max_drawdown': ANY, 'max_drawdown_abs': ANY, + 'trading_volume': expected['trading_volume'], } diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8c12d2ea0..357233dfa 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -2269,6 +2269,7 @@ def test_Trade_object_idem(): 'get_exit_reason_performance', 'get_enter_tag_performance', 'get_mix_tag_performance', + 'get_trading_volume', ) From 0809f9aef69776805912bf8b32621a0a3d481959 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Jun 2022 17:44:15 +0200 Subject: [PATCH 123/162] Add offset to trade response --- freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/rpc.py | 1 + tests/rpc/test_rpc_apiserver.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 7566e2ac0..333f2fe6e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -283,6 +283,7 @@ class OpenTradeSchema(TradeSchema): class TradeResponse(BaseModel): trades: List[TradeSchema] trades_count: int + offset: int total_trades: int diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 31fe4c469..dbbb78c98 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -365,6 +365,7 @@ class RPC: return { "trades": output, "trades_count": len(output), + "offset": offset, "total_trades": Trade.get_trades([Trade.is_open.is_(False)]).count(), } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b0ff5e1b2..c0de54c6d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -578,9 +578,10 @@ def test_api_trades(botclient, mocker, fee, markets, is_short): ) rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) - assert len(rc.json()) == 3 + assert len(rc.json()) == 4 assert rc.json()['trades_count'] == 0 assert rc.json()['total_trades'] == 0 + assert rc.json()['offset'] == 0 create_mock_trades(fee, is_short=is_short) Trade.query.session.flush() From 0d967f93baf69e08bda264df0702f4a433abf64b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jun 2022 16:13:00 +0200 Subject: [PATCH 124/162] Improve performance of some RPC calls These don't need orders to be loaded. As a side-effect, this will also reduce the strain on the database. --- freqtrade/persistence/trade_model.py | 13 +++++++++---- freqtrade/rpc/rpc.py | 5 +++-- tests/test_persistence.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 39ebd75b4..0c8c985c8 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func) -from sqlalchemy.orm import Query, relationship +from sqlalchemy.orm import Query, lazyload, relationship from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort from freqtrade.enums import ExitType, TradingMode @@ -1115,7 +1115,7 @@ class Trade(_DECL_BASE, LocalTrade): ) @staticmethod - def get_trades(trade_filter=None) -> Query: + def get_trades(trade_filter=None, include_orders: bool = True) -> Query: """ Helper function to query Trades using filters. NOTE: Not supported in Backtesting. @@ -1130,9 +1130,14 @@ class Trade(_DECL_BASE, LocalTrade): if trade_filter is not None: if not isinstance(trade_filter, list): trade_filter = [trade_filter] - return Trade.query.filter(*trade_filter) + this_query = Trade.query.filter(*trade_filter) else: - return Trade.query + this_query = Trade.query + if not include_orders: + # Don't load order relations + # Consider using noload or raiseload instead of lazyload + this_query = this_query.options(lazyload(Trade.orders)) + return this_query @staticmethod def get_open_order_trades() -> List['Trade']: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dbbb78c98..c42a6f683 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -380,7 +380,7 @@ class RPC: return 'losses' else: return 'draws' - trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)]) + trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False) # Sell reason exit_reasons = {} for trade in trades: @@ -408,7 +408,8 @@ class RPC: """ Returns cumulative profit statistics """ trade_filter = ((Trade.is_open.is_(False) & (Trade.close_date >= start_date)) | Trade.is_open.is_(True)) - trades: List[Trade] = Trade.get_trades(trade_filter).order_by(Trade.id).all() + trades: List[Trade] = Trade.get_trades( + trade_filter, include_orders=False).order_by(Trade.id).all() profit_all_coin = [] profit_all_ratio = [] diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 357233dfa..deaad258b 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -2075,6 +2075,24 @@ def test_get_trades_proxy(fee, use_db, is_short): Trade.use_db = True +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize('is_short', [True, False]) +def test_get_trades__query(fee, is_short): + query = Trade.get_trades([]) + # without orders there should be no join issued. + query1 = Trade.get_trades([], include_orders=False) + + assert "JOIN orders" in str(query) + assert "JOIN orders" not in str(query1) + + create_mock_trades(fee, is_short) + query = Trade.get_trades([]) + query1 = Trade.get_trades([], include_orders=False) + + assert "JOIN orders" in str(query) + assert "JOIN orders" not in str(query1) + + def test_get_trades_backtest(): Trade.use_db = False with pytest.raises(NotImplementedError, match=r"`Trade.get_trades\(\)` not .*"): From 8406010260e3d79be0638b375b63fd447d4711da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 03:01:26 +0000 Subject: [PATCH 125/162] Bump types-cachetools from 5.0.1 to 5.0.2 Bumps [types-cachetools](https://github.com/python/typeshed) from 5.0.1 to 5.0.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 19912d59c..53c85f176 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ time-machine==2.7.0 nbconvert==6.5.0 # mypy types -types-cachetools==5.0.1 +types-cachetools==5.0.2 types-filelock==3.2.7 types-requests==2.27.30 types-tabulate==0.8.9 From 55fb7656dfdafebf754d6a5b14c8bdd0eca2eb29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 06:58:41 +0200 Subject: [PATCH 126/162] Update pre-commit cachetools --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5c1a36f5..e057627cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: mypy exclude: build_helpers additional_dependencies: - - types-cachetools==5.0.1 + - types-cachetools==5.0.2 - types-filelock==3.2.7 - types-requests==2.27.30 - types-tabulate==0.8.9 From 0804fc7a3af91c25b291bd991132cf77d8d42946 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:01:35 +0200 Subject: [PATCH 127/162] CI should run ccxt tests only once --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 551268af7..2f67ec5fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: - name: Tests incl. ccxt compatibility tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun - if: matrix.python-version == '3.9' + if: matrix.python-version == '3.9' and matrix.os == 'ubuntu-22.04' - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.9') From f9668ede4a54edda390197b3261504b3d526c77d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:02:12 +0200 Subject: [PATCH 128/162] Fix CI Syntax error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f67ec5fe..81a18c4c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: - name: Tests incl. ccxt compatibility tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun - if: matrix.python-version == '3.9' and matrix.os == 'ubuntu-22.04' + if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-22.04' - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.9') From 50c19ece53644808adf80a95eebefdf6fe3f4c6d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:05:51 +0200 Subject: [PATCH 129/162] Fix ccxt test gateio flukyness --- tests/exchange/test_ccxt_compat.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index e016873cb..50154bcaf 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -199,8 +199,13 @@ class TestCCXTExchange(): l2 = exchange.fetch_l2_order_book(pair) assert 'asks' in l2 assert 'bids' in l2 + assert len(l2['asks']) >= 1 + assert len(l2['bids']) >= 1 l2_limit_range = exchange._ft_has['l2_limit_range'] l2_limit_range_required = exchange._ft_has['l2_limit_range_required'] + if exchangename == 'gateio': + # TODO: Gateio is unstable here at the moment, ignoring the limit partially. + return for val in [1, 2, 5, 25, 100]: l2 = exchange.fetch_l2_order_book(pair, val) if not l2_limit_range or val in l2_limit_range: From 996372b8f6c18b9721034bd6abfd41532f3e2b62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 05:06:39 +0000 Subject: [PATCH 130/162] Bump colorama from 0.4.4 to 0.4.5 Bumps [colorama](https://github.com/tartley/colorama) from 0.4.4 to 0.4.5. - [Release notes](https://github.com/tartley/colorama/releases) - [Changelog](https://github.com/tartley/colorama/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tartley/colorama/compare/0.4.4...0.4.5) --- updated-dependencies: - dependency-name: colorama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b2dbd921e..bd28e3768 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ aiofiles==0.8.0 psutil==5.9.1 # Support for colorized terminal output -colorama==0.4.4 +colorama==0.4.5 # Building config files interactively questionary==1.10.0 prompt-toolkit==3.0.29 From e1e3a903f98aebd0d2db6b9c6e0e5f51b70075c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 05:07:35 +0000 Subject: [PATCH 131/162] Bump ccxt from 1.87.12 to 1.88.15 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.87.12 to 1.88.15. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.87.12...1.88.15) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b2dbd921e..ec244806d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.87.12 +ccxt==1.88.15 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 1cd2b0504a9e1c684326ab59a6dcb6f2a8eb85a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 07:15:15 +0200 Subject: [PATCH 132/162] Run regular tests for 3.9 under other ubuntu systems --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81a18c4c9..818d250ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc - if: matrix.python-version != '3.9' + if: matrix.python-version != '3.9' && matrix.os != 'ubuntu-22.04' - name: Tests incl. ccxt compatibility tests run: | From 3189b284c014c4624ba27fc8abbde982c5b36c0c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Jun 2022 08:04:34 +0200 Subject: [PATCH 133/162] Fix tests condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 818d250ca..4fe1ad853 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Tests run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc - if: matrix.python-version != '3.9' && matrix.os != 'ubuntu-22.04' + if: matrix.python-version != '3.9' || matrix.os != 'ubuntu-22.04' - name: Tests incl. ccxt compatibility tests run: | From 15fac746a8129e00acb86246e16b00fb7daea89f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 06:59:58 +0000 Subject: [PATCH 134/162] Bump mkdocs-material from 8.3.4 to 8.3.6 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.3.4 to 8.3.6. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.3.4...8.3.6) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 1f342ca02..6477ad23f 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.3.4 +mkdocs-material==8.3.6 mdx_truly_sane_lists==1.2 pymdown-extensions==9.5 jinja2==3.1.2 From 53e5483daadfac741e21987be8f78b949ad6e808 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jun 2022 06:30:30 +0200 Subject: [PATCH 135/162] Store StopPrice for dry-run orders closes #6996 --- freqtrade/exchange/binance.py | 13 +++++++++---- freqtrade/exchange/gateio.py | 6 ++++-- freqtrade/exchange/huobi.py | 8 +++++++- freqtrade/exchange/kucoin.py | 5 ++++- freqtrade/persistence/migrations.py | 11 ++++++----- freqtrade/persistence/trade_model.py | 3 +++ tests/exchange/test_kucoin.py | 4 ++-- tests/test_persistence.py | 2 ++ 8 files changed, 37 insertions(+), 15 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 03546dcf9..37a3c419d 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -52,10 +52,15 @@ class Binance(Exchange): ordertype = 'stop' if self.trading_mode == TradingMode.FUTURES else 'stop_loss_limit' - return order['type'] == ordertype and ( - (side == "sell" and stop_loss > float(order['stopPrice'])) or - (side == "buy" and stop_loss < float(order['stopPrice'])) - ) + return ( + order.get('stopPrice', None) is None + or ( + order['type'] == ordertype + and ( + (side == "sell" and stop_loss > float(order['stopPrice'])) or + (side == "buy" and stop_loss < float(order['stopPrice'])) + ) + )) def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict: tickers = super().get_tickers(symbols=symbols, cached=cached) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index fd9a2b2b3..bf50167da 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -114,5 +114,7 @@ class Gateio(Exchange): Verify stop_loss against stoploss-order value (limit or price) Returns True if adjustment is necessary. """ - return ((side == "sell" and stop_loss > float(order['stopPrice'])) or - (side == "buy" and stop_loss < float(order['stopPrice']))) + return (order.get('stopPrice', None) is None or ( + side == "sell" and stop_loss > float(order['stopPrice'])) or + (side == "buy" and stop_loss < float(order['stopPrice'])) + ) diff --git a/freqtrade/exchange/huobi.py b/freqtrade/exchange/huobi.py index 71c4d1cf6..736515dec 100644 --- a/freqtrade/exchange/huobi.py +++ b/freqtrade/exchange/huobi.py @@ -27,7 +27,13 @@ class Huobi(Exchange): Verify stop_loss against stoploss-order value (limit or price) Returns True if adjustment is necessary. """ - return order['type'] == 'stop' and stop_loss > float(order['stopPrice']) + return ( + order.get('stopPrice', None) is None + or ( + order['type'] == 'stop' + and stop_loss > float(order['stopPrice']) + ) + ) def _get_stop_params(self, ordertype: str, stop_price: float) -> Dict: diff --git a/freqtrade/exchange/kucoin.py b/freqtrade/exchange/kucoin.py index f23189b3c..21eaa4bc3 100644 --- a/freqtrade/exchange/kucoin.py +++ b/freqtrade/exchange/kucoin.py @@ -33,7 +33,10 @@ class Kucoin(Exchange): Verify stop_loss against stoploss-order value (limit or price) Returns True if adjustment is necessary. """ - return order['info'].get('stop') is not None and stop_loss > float(order['stopPrice']) + return ( + order.get('stopPrice', None) is None + or stop_loss > float(order['stopPrice']) + ) def _get_stop_params(self, ordertype: str, stop_price: float) -> Dict: diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index b0fdf0412..f8fc5d619 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -201,16 +201,18 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null') average = get_column_def(cols_order, 'average', 'null') + stop_price = get_column_def(cols_order, 'stop_price', 'null') # sqlite does not support literals for booleans with engine.begin() as connection: connection.execute(text(f""" insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, average, remaining, cost, - order_date, order_filled_date, order_update_date, ft_fee_base) + stop_price, order_date, order_filled_date, order_update_date, ft_fee_base) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, {average} average, remaining, - cost, order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base + cost, {stop_price} stop_price, order_date, order_filled_date, + order_update_date, {ft_fee_base} ft_fee_base from {table_back_name} """)) @@ -294,9 +296,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # Check if migration necessary # Migrates both trades and orders table! - # if ('orders' not in previous_tables - # or not has_column(cols_orders, 'leverage')): - if not has_column(cols_trades, 'base_currency'): + if not has_column(cols_orders, 'stop_price'): + # if not has_column(cols_trades, 'base_currency'): logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") migrate_trades_and_orders_table( diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 0c8c985c8..324002685 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -57,6 +57,7 @@ class Order(_DECL_BASE): filled = Column(Float, nullable=True) remaining = Column(Float, nullable=True) cost = Column(Float, nullable=True) + stop_price = Column(Float, nullable=True) order_date = Column(DateTime, nullable=True, default=datetime.utcnow) order_filled_date = Column(DateTime, nullable=True) order_update_date = Column(DateTime, nullable=True) @@ -107,6 +108,7 @@ class Order(_DECL_BASE): self.average = order.get('average', self.average) self.remaining = order.get('remaining', self.remaining) self.cost = order.get('cost', self.cost) + self.stop_price = order.get('stopPrice', self.stop_price) if 'timestamp' in order and order['timestamp'] is not None: self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc) @@ -130,6 +132,7 @@ class Order(_DECL_BASE): 'side': self.ft_order_side, 'filled': self.filled, 'remaining': self.remaining, + 'stopPrice': self.stop_price, 'datetime': self.order_date_utc.strftime('%Y-%m-%dT%H:%M:%S.%f'), 'timestamp': int(self.order_date_utc.timestamp() * 1000), 'status': self.status, diff --git a/tests/exchange/test_kucoin.py b/tests/exchange/test_kucoin.py index 8af1e83a3..ebaf5ae81 100644 --- a/tests/exchange/test_kucoin.py +++ b/tests/exchange/test_kucoin.py @@ -123,5 +123,5 @@ def test_stoploss_adjust_kucoin(mocker, default_conf): assert exchange.stoploss_adjust(1501, order, 'sell') assert not exchange.stoploss_adjust(1499, order, 'sell') # Test with invalid order case - order['info']['stop'] = None - assert not exchange.stoploss_adjust(1501, order, 'sell') + order['stopPrice'] = None + assert exchange.stoploss_adjust(1501, order, 'sell') diff --git a/tests/test_persistence.py b/tests/test_persistence.py index deaad258b..c52e06c82 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -2717,5 +2717,7 @@ def test_order_to_ccxt(limit_buy_order_open): del raw_order['fee'] del raw_order['datetime'] del raw_order['info'] + assert raw_order['stopPrice'] is None + del raw_order['stopPrice'] del limit_buy_order_open['datetime'] assert raw_order == limit_buy_order_open From 90feccf33c4d554c0ea8cbfe31a8b9419b8b24f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jun 2022 07:17:24 +0200 Subject: [PATCH 136/162] slightly update custom dockerfile with add. comment closes #6994 --- docker/Dockerfile.custom | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.custom b/docker/Dockerfile.custom index 3b55fcb0e..6e321f14d 100644 --- a/docker/Dockerfile.custom +++ b/docker/Dockerfile.custom @@ -7,4 +7,5 @@ FROM freqtradeorg/freqtrade:develop # The below dependency - pyti - serves as an example. Please use whatever you need! RUN pip install --user pyti +# Switch back to user (only if you required root above) # USER ftuser From ddc355feb6758f019bb0cdbfb9dfe76bdadd34da Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jun 2022 08:07:22 +0000 Subject: [PATCH 137/162] Bump numpy from 1.22.4 to 1.23.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 40b5d660d..b62238024 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.22.4 +numpy==1.23.0 pandas==1.4.2 pandas-ta==0.3.14b From 2b07d346118b98bd976200233c57687e9ca28199 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jun 2022 20:47:51 +0200 Subject: [PATCH 138/162] Revert several undesired changes --- freqtrade/exchange/exchange.py | 8 ++++---- freqtrade/freqtradebot.py | 6 +++--- freqtrade/optimize/backtesting.py | 2 +- freqtrade/plugins/pairlist/PriceFilter.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 4fa8f1def..4febe5652 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -383,8 +383,8 @@ class Exchange: Ensures that Configured mode aligns to """ return ( - market.get('quote') is not None - and market.get('base') is not None + market.get('quote', None) is not None + and market.get('base', None) is not None and (self.precisionMode != TICK_SIZE # Too low precision will falsify calculations or market.get('precision', {}).get('price') > 1e-11) @@ -1599,7 +1599,7 @@ class Exchange: def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, price: float = 1, taker_or_maker: str = 'maker') -> float: try: - if self._config['dry_run'] and self._config.get('fee') is not None: + if self._config['dry_run'] and self._config.get('fee', None) is not None: return self._config['fee'] # validate that markets are loaded before trying to get fee if self._api.markets is None or len(self._api.markets) == 0: @@ -1660,7 +1660,7 @@ class Exchange: fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') except ExchangeError: - fee_to_quote_rate = self._config['exchange'].get('unknown_fee_rate') + fee_to_quote_rate = self._config['exchange'].get('unknown_fee_rate', None) if not fee_to_quote_rate: return None return round((self._contracts_to_amount( diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index affc6f360..ee535e9c3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -855,7 +855,7 @@ class FreqtradeBot(LoggingMixin): 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency'), + 'fiat_currency': self.config.get('fiat_display_currency', None), 'amount': safe_value_fallback(order, 'filled', 'amount') or trade.amount, 'open_date': trade.open_date or datetime.utcnow(), 'current_rate': current_rate, @@ -884,7 +884,7 @@ class FreqtradeBot(LoggingMixin): 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency'), + 'fiat_currency': self.config.get('fiat_display_currency', None), 'amount': trade.amount, 'open_date': trade.open_date, 'current_rate': current_rate, @@ -1590,7 +1590,7 @@ class FreqtradeBot(LoggingMixin): 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.now(timezone.utc), 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency'), + 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, } diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6f566a346..cacb87745 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -123,7 +123,7 @@ class Backtesting: if len(self.pairlists.whitelist) == 0: raise OperationalException("No pair in whitelist.") - if config.get('fee') is not None: + if config.get('fee', None) is not None: self.fee = config['fee'] else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index 4c5db52b1..009789eaf 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -70,7 +70,7 @@ class PriceFilter(IPairList): :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ - if ticker.get('last') is None or ticker.get('last') == 0: + if ticker.get('last', None) is None or ticker.get('last') == 0: self.log_once(f"Removed {pair} from whitelist, because " "ticker['last'] is empty (Usually no trade in the last 24h).", logger.info) From b5d0bc997d6c03b5e8dd0a2a10de4ec58062b2a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Jun 2022 17:25:33 +0200 Subject: [PATCH 139/162] Clarify stoploss behavior when not defining offset closes #6828 --- docs/stoploss.md | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 83f787947..6ddb485a4 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -130,7 +130,7 @@ In summary: The stoploss will be adjusted to be always be -10% of the highest ob ### Trailing stop loss, custom positive loss -It is also possible to have a default stop loss, when you are in the red with your buy (buy - fee), but once you hit positive result the system will utilize a new stop loss, which can have a different value. +You could also have a default stop loss when you are in the red with your buy (buy - fee), but once you hit a positive result (or an offset you define) the system will utilize a new stop loss, which can have a different value. For example, your default stop loss is -10%, but once you have more than 0% profit (example 0.1%) a different trailing stoploss will be used. !!! Note @@ -142,6 +142,8 @@ Both values require `trailing_stop` to be set to true and `trailing_stop_positiv stoploss = -0.10 trailing_stop = True trailing_stop_positive = 0.02 + trailing_stop_positive_offset = 0.0 + trailing_only_offset_is_reached = False # Default - not necessary for this example ``` For example, simplified math: @@ -156,11 +158,31 @@ For example, simplified math: The 0.02 would translate to a -2% stop loss. Before this, `stoploss` is used for the trailing stoploss. +!!! Tip "Use an offset to change your stoploss" + Use `trailing_stop_positive_offset` to ensure that your new trailing stoploss will be in profit by setting `trailing_stop_positive_offset` higher than `trailing_stop_positive`. Your first new stoploss value will then already have locked in profits. + + Example with simplified math: + + ``` python + stoploss = -0.10 + trailing_stop = True + trailing_stop_positive = 0.02 + trailing_stop_positive_offset = 0.03 + ``` + + * the bot buys an asset at a price of 100$ + * the stop loss is defined at -10%, so the stop loss would get triggered once the asset drops below 90$ + * assuming the asset now increases to 102$ + * the stoploss will now be at 91.8$ - 10% below the highest observed rate + * assuming the asset now increases to 103.5$ (above the offset configured) + * the stop loss will now be -2% of 103$ = 101.42$ + * now the asset drops in value to 102\$, the stop loss will still be 101.42$ and would trigger once price breaks below 101.42$ + ### Trailing stop loss only once the trade has reached a certain offset -It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market turns. +You can also keep a static stoploss until the offset is reached, and then trail the trade to take profits once the market turns. -If `"trailing_only_offset_is_reached": true` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured `stoploss`. +If `trailing_only_offset_is_reached = True` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured `stoploss`. This option can be used with or without `trailing_stop_positive`, but uses `trailing_stop_positive_offset` as offset. ``` python @@ -203,7 +225,6 @@ If price moves 1% - you've lost 10$ of your own capital - therfore stoploss will Make sure to be aware of this, and avoid using too tight stoploss (at 10x leverage, 10% risk may be too little to allow the trade to "breath" a little). - ## Changing stoploss on open trades A stoploss on an open trade can be changed by changing the value in the configuration or strategy and use the `/reload_config` command (alternatively, completely stopping and restarting the bot also works). From 92dbb0d3660efd602e9851f7b9c35126fc5b5620 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 03:02:07 +0000 Subject: [PATCH 140/162] Bump uvicorn from 0.17.6 to 0.18.1 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.17.6 to 0.18.1. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/uvicorn/compare/0.17.6...0.18.1) --- updated-dependencies: - dependency-name: uvicorn dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b62238024..7781b43e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ sdnotify==0.3.2 # API Server fastapi==0.78.0 -uvicorn==0.17.6 +uvicorn==0.18.1 pyjwt==2.4.0 aiofiles==0.8.0 psutil==5.9.1 From 4840c7d2fd9f98757fe76ebe90f75f14afb5b814 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 03:02:16 +0000 Subject: [PATCH 141/162] Bump pytest-mock from 3.7.0 to 3.8.1 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 3.7.0 to 3.8.1. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v3.7.0...v3.8.1) --- updated-dependencies: - dependency-name: pytest-mock dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 53c85f176..29e21e116 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,7 +12,7 @@ pre-commit==2.19.0 pytest==7.1.2 pytest-asyncio==0.18.3 pytest-cov==3.0.0 -pytest-mock==3.7.0 +pytest-mock==3.8.1 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking From 45db2347dcd4545fa882cd32fffc9f5d9b936640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 03:02:29 +0000 Subject: [PATCH 142/162] Bump mkdocs-material from 8.3.6 to 8.3.8 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.3.6 to 8.3.8. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.3.6...8.3.8) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 6477ad23f..fe00705b9 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.3.6 +mkdocs-material==8.3.8 mdx_truly_sane_lists==1.2 pymdown-extensions==9.5 jinja2==3.1.2 From 963f38a69077524a7ce60e2099ae2c641f57c4a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 03:02:46 +0000 Subject: [PATCH 143/162] Bump sqlalchemy from 1.4.37 to 1.4.39 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.37 to 1.4.39. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b62238024..7e1bc79e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.88.15 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 -SQLAlchemy==1.4.37 +SQLAlchemy==1.4.39 python-telegram-bot==13.12 arrow==1.2.2 cachetools==4.2.2 From 8b7dc031f7db50a35731d4b8d92d9196153ae639 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 03:02:51 +0000 Subject: [PATCH 144/162] Bump plotly from 5.8.2 to 5.9.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.8.2 to 5.9.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.8.2...v5.9.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index a2a894c57..0f6ae94c2 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.8.2 +plotly==5.9.0 From efee148e43e21d0bc90af95ab8e7635bd5624502 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 03:02:53 +0000 Subject: [PATCH 145/162] Bump types-cachetools from 5.0.2 to 5.2.1 Bumps [types-cachetools](https://github.com/python/typeshed) from 5.0.2 to 5.2.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 53c85f176..c3edec904 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ time-machine==2.7.0 nbconvert==6.5.0 # mypy types -types-cachetools==5.0.2 +types-cachetools==5.2.1 types-filelock==3.2.7 types-requests==2.27.30 types-tabulate==0.8.9 From 6510c8d330efedd5f7dad2d48bf566db211f4ce3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 03:03:00 +0000 Subject: [PATCH 146/162] Bump tabulate from 0.8.9 to 0.8.10 Bumps [tabulate](https://github.com/astanin/python-tabulate) from 0.8.9 to 0.8.10. - [Release notes](https://github.com/astanin/python-tabulate/releases) - [Changelog](https://github.com/astanin/python-tabulate/blob/master/CHANGELOG) - [Commits](https://github.com/astanin/python-tabulate/compare/v0.8.9...v0.8.10) --- updated-dependencies: - dependency-name: tabulate dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b62238024..8f63c5add 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ urllib3==1.26.9 jsonschema==4.6.0 TA-Lib==0.4.24 technical==1.3.0 -tabulate==0.8.9 +tabulate==0.8.10 pycoingecko==2.2.0 jinja2==3.1.2 tables==3.7.0 From 0ef2c812db844c9706116ce5b0381e090e62c1e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 05:38:31 +0000 Subject: [PATCH 147/162] Bump ccxt from 1.88.15 to 1.89.14 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.88.15 to 1.89.14. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.88.15...1.89.14) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7e1bc79e9..daf8ebd6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.0 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.88.15 +ccxt==1.89.14 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 9a9d1a89742036169332b4fe37bdeec1dd02293c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 05:39:04 +0000 Subject: [PATCH 148/162] Bump orjson from 3.7.2 to 3.7.3 Bumps [orjson](https://github.com/ijl/orjson) from 3.7.2 to 3.7.3. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.7.2...3.7.3) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7e1bc79e9..cf47a80ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ urllib3==1.26.9 jsonschema==4.6.0 TA-Lib==0.4.24 technical==1.3.0 -tabulate==0.8.9 +tabulate==0.8.10 pycoingecko==2.2.0 jinja2==3.1.2 tables==3.7.0 @@ -28,7 +28,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.6 # Properly format api responses -orjson==3.7.2 +orjson==3.7.3 # Notify systemd sdnotify==0.3.2 From 01185ab48391cf702f187a4dee47417de699ae9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 Jun 2022 07:59:26 +0200 Subject: [PATCH 149/162] update cachetools precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e057627cf..983df6fad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: mypy exclude: build_helpers additional_dependencies: - - types-cachetools==5.0.2 + - types-cachetools==5.2.1 - types-filelock==3.2.7 - types-requests==2.27.30 - types-tabulate==0.8.9 From 82ef97af7e16c82ae9ba342fd5b4e945bb590649 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 07:44:33 +0000 Subject: [PATCH 150/162] Bump pandas from 1.4.2 to 1.4.3 Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.4.2 to 1.4.3. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/main/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.4.2...v1.4.3) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e64ddca05..178d852b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.23.0 -pandas==1.4.2 +pandas==1.4.3 pandas-ta==0.3.14b ccxt==1.89.14 From d60127a6d8b159ae56e3a4d1ff13a9e225cc0b0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:59:07 +0000 Subject: [PATCH 151/162] Bump time-machine from 2.7.0 to 2.7.1 Bumps [time-machine](https://github.com/adamchainz/time-machine) from 2.7.0 to 2.7.1. - [Release notes](https://github.com/adamchainz/time-machine/releases) - [Changelog](https://github.com/adamchainz/time-machine/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/time-machine/compare/2.7.0...2.7.1) --- updated-dependencies: - dependency-name: time-machine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ebfbafe07..03773faf6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ pytest-mock==3.8.1 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.7.0 +time-machine==2.7.1 # Convert jupyter notebooks to markdown documents nbconvert==6.5.0 From f6e058a327a5e9a3db6e388266d79879186e2cb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:59:19 +0000 Subject: [PATCH 152/162] Bump types-requests from 2.27.30 to 2.28.0 Bumps [types-requests](https://github.com/python/typeshed) from 2.27.30 to 2.28.0. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ebfbafe07..925bf1929 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,6 +24,6 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.27.30 +types-requests==2.28.0 types-tabulate==0.8.9 types-python-dateutil==2.8.17 From 0c69a0886392114ca65981c923fa357bf53ebd07 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 Jun 2022 12:09:27 +0200 Subject: [PATCH 153/162] update requests precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 983df6fad..958312d6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.27.30 + - types-requests==2.28.0 - types-tabulate==0.8.9 - types-python-dateutil==2.8.17 # stages: [push] From 8b1798522c17b2f651d4d0653247458ff51709da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 13:18:58 +0000 Subject: [PATCH 154/162] Bump types-tabulate from 0.8.9 to 0.8.11 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.8.9 to 0.8.11. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 852890979..16bb8c47b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -25,5 +25,5 @@ nbconvert==6.5.0 types-cachetools==5.2.1 types-filelock==3.2.7 types-requests==2.28.0 -types-tabulate==0.8.9 +types-tabulate==0.8.11 types-python-dateutil==2.8.17 From 74471e41dbceda78e829e2e377a2773617732e47 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 Jun 2022 18:23:00 +0200 Subject: [PATCH 155/162] update tabulate precommit types --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 958312d6c..71e230a39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - types-cachetools==5.2.1 - types-filelock==3.2.7 - types-requests==2.28.0 - - types-tabulate==0.8.9 + - types-tabulate==0.8.11 - types-python-dateutil==2.8.17 # stages: [push] From f2bc35e058cd608a8100c67ed32f626a49451ae9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 20:06:56 +0000 Subject: [PATCH 156/162] Bump types-python-dateutil from 2.8.17 to 2.8.18 Bumps [types-python-dateutil](https://github.com/python/typeshed) from 2.8.17 to 2.8.18. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 16bb8c47b..50cb98fa1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,4 +26,4 @@ types-cachetools==5.2.1 types-filelock==3.2.7 types-requests==2.28.0 types-tabulate==0.8.11 -types-python-dateutil==2.8.17 +types-python-dateutil==2.8.18 From 86f407702419cb23e2c0aa127b1646daed71d363 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Jun 2022 07:37:54 +0200 Subject: [PATCH 157/162] update dateutil precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71e230a39..59e7f6894 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - types-filelock==3.2.7 - types-requests==2.28.0 - types-tabulate==0.8.11 - - types-python-dateutil==2.8.17 + - types-python-dateutil==2.8.18 # stages: [push] - repo: https://github.com/pycqa/isort From 906c7b92fe997c0cb8fcbec3f450b1a601479dba Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jul 2022 11:05:15 +0200 Subject: [PATCH 158/162] Add enhance testcase to show problematic exit_reason behavior --- tests/test_freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 4f3d5f667..4963e2b0a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3951,9 +3951,9 @@ def test_ignore_roi_if_entry_signal(default_conf_usdt, limit_order, limit_order_ # Test if entry-signal is absent (should sell due to roi = true) if is_short: - patch_get_signal(freqtrade, enter_long=False, exit_short=False) + patch_get_signal(freqtrade, enter_long=False, exit_short=False, exit_tag='something') else: - patch_get_signal(freqtrade, enter_long=False, exit_long=False) + patch_get_signal(freqtrade, enter_long=False, exit_long=False, exit_tag='something') assert freqtrade.handle_trade(trade) is True assert trade.exit_reason == ExitType.ROI.value From f2fdc21374beee7e06f2d3077c5c3f33b5080a3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jul 2022 11:07:05 +0200 Subject: [PATCH 159/162] Only use exit_tag if exit_type i exit_signal closes #7027 --- freqtrade/freqtradebot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ee535e9c3..db81a81ba 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1125,9 +1125,10 @@ class FreqtradeBot(LoggingMixin): ) for should_exit in exits: if should_exit.exit_flag: + exit_tag1 = exit_tag if should_exit.exit_type == ExitType.EXIT_SIGNAL else None logger.info(f'Exit for {trade.pair} detected. Reason: {should_exit.exit_type}' - f'{f" Tag: {exit_tag}" if exit_tag is not None else ""}') - exited = self.execute_trade_exit(trade, exit_rate, should_exit, exit_tag=exit_tag) + f'{f" Tag: {exit_tag1}" if exit_tag1 is not None else ""}') + exited = self.execute_trade_exit(trade, exit_rate, should_exit, exit_tag=exit_tag1) if exited: return True return False From c5e6520fee43582496efa2c28b9b21ba2d403235 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jul 2022 13:35:26 +0200 Subject: [PATCH 160/162] Reorder methods in freqtradebot --- freqtrade/freqtradebot.py | 46 +++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index db81a81ba..b30c9b965 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -959,6 +959,29 @@ class FreqtradeBot(LoggingMixin): logger.debug(f'Found no {exit_signal_type} signal for %s.', trade) return False + def _check_and_execute_exit(self, trade: Trade, exit_rate: float, + enter: bool, exit_: bool, exit_tag: Optional[str]) -> bool: + """ + Check and execute trade exit + """ + exits: List[ExitCheckTuple] = self.strategy.should_exit( + trade, + exit_rate, + datetime.now(timezone.utc), + enter=enter, + exit_=exit_, + force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 + ) + for should_exit in exits: + if should_exit.exit_flag: + exit_tag1 = exit_tag if should_exit.exit_type == ExitType.EXIT_SIGNAL else None + logger.info(f'Exit for {trade.pair} detected. Reason: {should_exit.exit_type}' + f'{f" Tag: {exit_tag1}" if exit_tag1 is not None else ""}') + exited = self.execute_trade_exit(trade, exit_rate, should_exit, exit_tag=exit_tag1) + if exited: + return True + return False + def create_stoploss_order(self, trade: Trade, stop_price: float) -> bool: """ Abstracts creating stoploss orders from the logic. @@ -1110,29 +1133,6 @@ class FreqtradeBot(LoggingMixin): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") - def _check_and_execute_exit(self, trade: Trade, exit_rate: float, - enter: bool, exit_: bool, exit_tag: Optional[str]) -> bool: - """ - Check and execute trade exit - """ - exits: List[ExitCheckTuple] = self.strategy.should_exit( - trade, - exit_rate, - datetime.now(timezone.utc), - enter=enter, - exit_=exit_, - force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 - ) - for should_exit in exits: - if should_exit.exit_flag: - exit_tag1 = exit_tag if should_exit.exit_type == ExitType.EXIT_SIGNAL else None - logger.info(f'Exit for {trade.pair} detected. Reason: {should_exit.exit_type}' - f'{f" Tag: {exit_tag1}" if exit_tag1 is not None else ""}') - exited = self.execute_trade_exit(trade, exit_rate, should_exit, exit_tag=exit_tag1) - if exited: - return True - return False - def manage_open_orders(self) -> None: """ Management of open orders on exchange. Unfilled orders might be cancelled if timeout From 07aa372e2a10b2afe3d899d4da3e63c004513788 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jul 2022 14:10:08 +0200 Subject: [PATCH 161/162] Ensure bot_loop_start is called in hyperopt, too closes #7001 --- docs/bot-basics.md | 5 ++++- freqtrade/optimize/backtesting.py | 3 +-- tests/optimize/test_hyperopt.py | 1 + tests/strategy/strats/hyperoptable_strategy.py | 5 +++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 1acbca565..14823722e 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -20,7 +20,9 @@ All profit calculations of Freqtrade include fees. For Backtesting / Hyperopt / ## Bot execution logic Starting freqtrade in dry-run or live mode (using `freqtrade trade`) will start the bot and start the bot iteration loop. -By default, loop runs every few seconds (`internals.process_throttle_secs`) and does roughly the following in the following sequence: +This will also run the `bot_start()` callback. + +By default, the bot loop runs every few seconds (`internals.process_throttle_secs`) and performs the following actions: * Fetch open trades from persistence. * Calculate current list of tradable pairs. @@ -54,6 +56,7 @@ This loop will be repeated again and again until the bot is stopped. [backtesting](backtesting.md) or [hyperopt](hyperopt.md) do only part of the above logic, since most of the trading operations are fully simulated. * Load historic data for configured pairlist. +* Calls `bot_start()` once. * Calls `bot_loop_start()` once. * Calculate indicators (calls `populate_indicators()` once per pair). * Calculate entry / exit signals (calls `populate_entry_trend()` and `populate_exit_trend()` once per pair). diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cacb87745..030d7bdf0 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -189,6 +189,7 @@ class Backtesting: self.strategy.order_types['stoploss_on_exchange'] = False self.strategy.ft_bot_start() + strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() def _load_protections(self, strategy: IStrategy): if self.config.get('enable_protections', False): @@ -1140,8 +1141,6 @@ class Backtesting: backtest_start_time = datetime.now(timezone.utc) self._set_strategy(strat) - strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() - # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): # Must come from strategy config, as the strategy may modify this setting. diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 9f3c5845f..1ad8b33cf 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -861,6 +861,7 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: hyperopt.backtesting.exchange.get_max_leverage = MagicMock(return_value=1.0) assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter) + assert hyperopt.backtesting.strategy.bot_loop_started is True assert hyperopt.backtesting.strategy.buy_rsi.in_space is True assert hyperopt.backtesting.strategy.buy_rsi.value == 35 diff --git a/tests/strategy/strats/hyperoptable_strategy.py b/tests/strategy/strats/hyperoptable_strategy.py index 28ecf617a..876b31b14 100644 --- a/tests/strategy/strats/hyperoptable_strategy.py +++ b/tests/strategy/strats/hyperoptable_strategy.py @@ -44,6 +44,11 @@ class HyperoptableStrategy(StrategyTestV2): }) return prot + bot_loop_started = False + + def bot_loop_start(self): + self.bot_loop_started = True + def bot_start(self, **kwargs) -> None: """ Parameters can also be defined here ... From c1d407851825c4565996860843e669f3b7184353 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jul 2022 15:04:38 +0200 Subject: [PATCH 162/162] Version bump to 2022.6 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 4a7da02b0..253e7d7ce 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.5.1' +__version__ = '2022.6' if 'dev' in __version__: try: