From 70e9fa6136a0d7705dfe6a72e0315467d6c0a838 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Tue, 27 Dec 2022 20:14:39 +0100 Subject: [PATCH 01/30] implementing the strategy_updater in a first version --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 14 +- freqtrade/commands/strategy_utils_commands.py | 37 +++ freqtrade/strategy/strategy_updater.py | 213 ++++++++++++++++++ 4 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 freqtrade/commands/strategy_utils_commands.py create mode 100644 freqtrade/strategy/strategy_updater.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 788657cc8..66a9c995b 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -22,5 +22,6 @@ from freqtrade.commands.optimize_commands import (start_backtesting, start_backt start_edge, start_hyperopt) from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit +from freqtrade.commands.strategy_utils_commands import start_strategy_update from freqtrade.commands.trade_commands import start_trading from freqtrade.commands.webserver_commands import start_webserver diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index b53a1022d..9990ad230 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -115,6 +115,8 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] +ARGS_STRATEGY_UTILS = ARGS_COMMON_OPTIMIZE + ["strategy_list"] + class Arguments: """ @@ -198,8 +200,8 @@ class Arguments: start_list_freqAI_models, start_list_markets, start_list_strategies, start_list_timeframes, start_new_config, start_new_strategy, start_plot_dataframe, - start_plot_profit, start_show_trades, start_test_pairlist, - start_trading, start_webserver) + start_plot_profit, start_show_trades, start_strategy_update, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -440,3 +442,11 @@ class Arguments: parents=[_common_parser]) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) + + # Add strategy_updater subcommand + strategy_updater_cmd = subparsers.add_parser('strategy-updater', + help='updates outdated strategy' + 'files to the current version', + parents=[_common_parser]) + strategy_updater_cmd.set_defaults(func=start_strategy_update) + self._build_args(optionlist=ARGS_STRATEGY_UTILS, parser=strategy_updater_cmd) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py new file mode 100644 index 000000000..b9431fecf --- /dev/null +++ b/freqtrade/commands/strategy_utils_commands.py @@ -0,0 +1,37 @@ +import logging +from typing import Any, Dict + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode +from freqtrade.resolvers import StrategyResolver + + +logger = logging.getLogger(__name__) + + +def start_strategy_update(args: Dict[str, Any]) -> None: + """ + Start the strategy updating script + :param args: Cli args from Arguments() + :return: None + """ + + # Import here to avoid loading backtesting module when it's not used + from freqtrade.strategy.strategy_updater import strategy_updater + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=True, recursive=config.get('recursive_strategy_search', False)) + + filtered_strategy_objs = [] + for args_strategy in args['strategy_list']: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == args_strategy and strategy_obj not in filtered_strategy_objs: + filtered_strategy_objs.append(strategy_obj) + break + + for filtered_strategy_obj in filtered_strategy_objs: + # Initialize backtesting object + instance_strategy_updater = strategy_updater() + strategy_updater.start(instance_strategy_updater, filtered_strategy_obj) diff --git a/freqtrade/strategy/strategy_updater.py b/freqtrade/strategy/strategy_updater.py new file mode 100644 index 000000000..fea8edf31 --- /dev/null +++ b/freqtrade/strategy/strategy_updater.py @@ -0,0 +1,213 @@ +import ast +import os +import shutil + + +class strategy_updater: + name_mapping = { + 'ticker_interval': 'timeframe', + 'buy': 'enter_long', + 'sell': 'exit_long', + 'buy_tag': 'enter_tag', + 'sell_reason': 'exit_reason', + + 'sell_signal': 'exit_signal', + 'custom_sell': 'custom_exit', + 'force_sell': 'force_exit', + 'emergency_sell': 'emergency_exit', + + # Strategy/config settings: + 'use_sell_signal': 'use_exit_signal', + 'sell_profit_only': 'exit_profit_only', + 'sell_profit_offset': 'exit_profit_offset', + 'ignore_roi_if_buy_signal': 'ignore_roi_if_entry_signal', + 'forcebuy_enable': 'force_entry_enable', + } + + function_mapping = { + 'populate_buy_trend': 'populate_entry_trend', + 'populate_sell_trend': 'populate_exit_trend', + 'custom_sell': 'custom_exit', + 'check_buy_timeout': 'check_entry_timeout', + 'check_sell_timeout': 'check_exit_timeout', + # '': '', + } + # order_time_in_force, order_types, unfilledtimeout + otif_ot_unfilledtimeout = { + 'buy': 'entry', + 'sell': 'exit', + } + + # create a dictionary that maps the old column names to the new ones + rename_dict = {'buy': 'enter_long', 'sell': 'exit_long', 'buy_tag': 'enter_tag'} + + def start(self, strategy_obj: dict) -> None: + """ + Run strategy updater + It updates a strategy to v3 with the help of the ast-module + :return: None + """ + + self.cwd = os.getcwd() + self.strategies_backup_folder = f'{os.getcwd()}user_data/strategies_orig_updater' + source_file = strategy_obj['location'] + + # read the file + with open(source_file, 'r') as f: + old_code = f.read() + if not os.path.exists(self.strategies_backup_folder): + os.makedirs(self.strategies_backup_folder) + + # backup original + # => currently no date after the filename, + # could get overridden pretty fast if this is fired twice! + shutil.copy(source_file, f"{self.strategies_backup_folder}/{strategy_obj['location_rel']}") + + # update the code + new_code = strategy_updater.update_code(self, old_code, + strategy_updater.name_mapping, + strategy_updater.function_mapping, + strategy_updater.rename_dict) + + # write the modified code to the destination folder + with open(source_file, 'w') as f: + f.write(new_code) + print(f"conversion of file {source_file} successful.") + + # define the function to update the code + def update_code(self, code, _name_mapping, _function_mapping, _rename_dict): + # parse the code into an AST + tree = ast.parse(code) + + # use the AST to update the code + updated_code = strategy_updater.modify_ast( + tree, + _name_mapping, + _function_mapping, + _rename_dict) + + # return the modified code without executing it + return updated_code + + # function that uses the ast module to update the code + def modify_ast(node, _name_mapping, _function_mapping, _rename_dict): # noqa + # create a visitor that will update the names and functions + class NameUpdater(ast.NodeTransformer): + def generic_visit(self, node): + # traverse the AST recursively by calling the visitor method for each child node + if hasattr(node, "_fields"): + for field_name, field_value in ast.iter_fields(node): + self.check_fields(field_value) + self.check_strategy_and_config_settings(field_value) + # add this check to handle the case where field_value is a slice + if isinstance(field_value, ast.Slice): + self.visit(field_value) + # add this check to handle the case where field_value is a target + if isinstance(field_value, ast.expr_context): + self.visit(field_value) + + def check_fields(self, field_value): + if isinstance(field_value, list): + for item in field_value: + if isinstance(item, ast.AST): + self.visit(item) + + def check_strategy_and_config_settings(self, field_value): + if (isinstance(field_value, ast.AST) and + hasattr(node, "targets") and + isinstance(node.targets, list)): + for target in node.targets: + if (hasattr(target, "id") and + (target.id == "order_time_in_force" or + target.id == "order_types" or + target.id == "unfilledtimeout") and + hasattr(field_value, "keys") and + isinstance(field_value.keys, list)): + for key in field_value.keys: + self.visit(key) + + def visit_Name(self, node): + # if the name is in the mapping, update it + if node.id in _name_mapping: + node.id = _name_mapping[node.id] + return node + + def visit_Import(self, node): + # do not update the names in import statements + return node + + def visit_ImportFrom(self, node): + # do not update the names in import statements + if hasattr(node, "module"): + if node.module == "freqtrade.strategy.hyper": + node.module = "freqtrade.strategy" + return node + + def visit_FunctionDef(self, node): + # if the function name is in the mapping, update it + if node.name in _function_mapping: + node.name = _function_mapping[node.name] + return self.generic_visit(node) + + def visit_Attribute(self, node): + # if the attribute name is 'nr_of_successful_buys', + # update it to 'nr_of_successful_entries' + if isinstance(node.value, ast.Name) and \ + node.value.id == 'trades' and \ + node.attr == 'nr_of_successful_buys': + node.attr = 'nr_of_successful_entries' + return self.generic_visit(node) + + def visit_ClassDef(self, node): + # check if the class is derived from IStrategy + if any(isinstance(base, ast.Name) and + base.id == 'IStrategy' for base in node.bases): + # check if the INTERFACE_VERSION variable exists + has_interface_version = any( + isinstance(child, ast.Assign) and + isinstance(child.targets[0], ast.Name) and + child.targets[0].id == 'INTERFACE_VERSION' + for child in node.body + ) + + # if the INTERFACE_VERSION variable does not exist, add it as the first child + if not has_interface_version: + node.body.insert(0, ast.parse('INTERFACE_VERSION = 3').body[0]) + # otherwise, update its value to 3 + else: + for child in node.body: + if isinstance(child, ast.Assign) and \ + isinstance(child.targets[0], ast.Name) and \ + child.targets[0].id == 'INTERFACE_VERSION': + child.value = ast.parse('3').body[0].value + return self.generic_visit(node) + + def visit_Subscript(self, node): + if isinstance(node.slice, ast.Constant): + if node.slice.value in strategy_updater.rename_dict: + # Replace the slice attributes with the values from rename_dict + node.slice.value = strategy_updater.rename_dict[node.slice.value] + if hasattr(node.slice, "elts"): + for elt in node.slice.elts: + if isinstance(elt, ast.Constant) and \ + elt.value in strategy_updater.rename_dict: + elt.value = strategy_updater.rename_dict[elt.value] + return node + + def visit_Constant(self, node): + # do not update the names in import statements + if node.value in \ + strategy_updater.otif_ot_unfilledtimeout: + node.value = \ + strategy_updater.otif_ot_unfilledtimeout[node.value] + return node + + # use the visitor to update the names and functions in the AST + NameUpdater().visit(node) + + # first fix the comments so it understands "\n" properly inside multi line comments. + ast.fix_missing_locations(node) + ast.increment_lineno(node, n=1) + + # generate the new code from the updated AST + return ast.unparse(node) From c6f045afa96c30bbd612af4b5877aeb41834d8fe Mon Sep 17 00:00:00 2001 From: hippocritical Date: Thu, 29 Dec 2022 22:17:52 +0100 Subject: [PATCH 02/30] fixing issues of the maintainer found a bug meaning elts could contain lists of elts (now recurively gone through) Next in line: writing tests based on StrategyUpdater.update_code --- freqtrade/commands/strategy_utils_commands.py | 6 +- freqtrade/strategy/strategy_updater.py | 213 ----------------- freqtrade/strategy/strategyupdater.py | 225 ++++++++++++++++++ 3 files changed, 228 insertions(+), 216 deletions(-) delete mode 100644 freqtrade/strategy/strategy_updater.py create mode 100644 freqtrade/strategy/strategyupdater.py diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index b9431fecf..13405089f 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -17,7 +17,7 @@ def start_strategy_update(args: Dict[str, Any]) -> None: """ # Import here to avoid loading backtesting module when it's not used - from freqtrade.strategy.strategy_updater import strategy_updater + from freqtrade.strategy.strategyupdater import StrategyUpdater config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -33,5 +33,5 @@ def start_strategy_update(args: Dict[str, Any]) -> None: for filtered_strategy_obj in filtered_strategy_objs: # Initialize backtesting object - instance_strategy_updater = strategy_updater() - strategy_updater.start(instance_strategy_updater, filtered_strategy_obj) + instance_strategy_updater = StrategyUpdater() + StrategyUpdater.start(instance_strategy_updater, config, filtered_strategy_obj) diff --git a/freqtrade/strategy/strategy_updater.py b/freqtrade/strategy/strategy_updater.py deleted file mode 100644 index fea8edf31..000000000 --- a/freqtrade/strategy/strategy_updater.py +++ /dev/null @@ -1,213 +0,0 @@ -import ast -import os -import shutil - - -class strategy_updater: - name_mapping = { - 'ticker_interval': 'timeframe', - 'buy': 'enter_long', - 'sell': 'exit_long', - 'buy_tag': 'enter_tag', - 'sell_reason': 'exit_reason', - - 'sell_signal': 'exit_signal', - 'custom_sell': 'custom_exit', - 'force_sell': 'force_exit', - 'emergency_sell': 'emergency_exit', - - # Strategy/config settings: - 'use_sell_signal': 'use_exit_signal', - 'sell_profit_only': 'exit_profit_only', - 'sell_profit_offset': 'exit_profit_offset', - 'ignore_roi_if_buy_signal': 'ignore_roi_if_entry_signal', - 'forcebuy_enable': 'force_entry_enable', - } - - function_mapping = { - 'populate_buy_trend': 'populate_entry_trend', - 'populate_sell_trend': 'populate_exit_trend', - 'custom_sell': 'custom_exit', - 'check_buy_timeout': 'check_entry_timeout', - 'check_sell_timeout': 'check_exit_timeout', - # '': '', - } - # order_time_in_force, order_types, unfilledtimeout - otif_ot_unfilledtimeout = { - 'buy': 'entry', - 'sell': 'exit', - } - - # create a dictionary that maps the old column names to the new ones - rename_dict = {'buy': 'enter_long', 'sell': 'exit_long', 'buy_tag': 'enter_tag'} - - def start(self, strategy_obj: dict) -> None: - """ - Run strategy updater - It updates a strategy to v3 with the help of the ast-module - :return: None - """ - - self.cwd = os.getcwd() - self.strategies_backup_folder = f'{os.getcwd()}user_data/strategies_orig_updater' - source_file = strategy_obj['location'] - - # read the file - with open(source_file, 'r') as f: - old_code = f.read() - if not os.path.exists(self.strategies_backup_folder): - os.makedirs(self.strategies_backup_folder) - - # backup original - # => currently no date after the filename, - # could get overridden pretty fast if this is fired twice! - shutil.copy(source_file, f"{self.strategies_backup_folder}/{strategy_obj['location_rel']}") - - # update the code - new_code = strategy_updater.update_code(self, old_code, - strategy_updater.name_mapping, - strategy_updater.function_mapping, - strategy_updater.rename_dict) - - # write the modified code to the destination folder - with open(source_file, 'w') as f: - f.write(new_code) - print(f"conversion of file {source_file} successful.") - - # define the function to update the code - def update_code(self, code, _name_mapping, _function_mapping, _rename_dict): - # parse the code into an AST - tree = ast.parse(code) - - # use the AST to update the code - updated_code = strategy_updater.modify_ast( - tree, - _name_mapping, - _function_mapping, - _rename_dict) - - # return the modified code without executing it - return updated_code - - # function that uses the ast module to update the code - def modify_ast(node, _name_mapping, _function_mapping, _rename_dict): # noqa - # create a visitor that will update the names and functions - class NameUpdater(ast.NodeTransformer): - def generic_visit(self, node): - # traverse the AST recursively by calling the visitor method for each child node - if hasattr(node, "_fields"): - for field_name, field_value in ast.iter_fields(node): - self.check_fields(field_value) - self.check_strategy_and_config_settings(field_value) - # add this check to handle the case where field_value is a slice - if isinstance(field_value, ast.Slice): - self.visit(field_value) - # add this check to handle the case where field_value is a target - if isinstance(field_value, ast.expr_context): - self.visit(field_value) - - def check_fields(self, field_value): - if isinstance(field_value, list): - for item in field_value: - if isinstance(item, ast.AST): - self.visit(item) - - def check_strategy_and_config_settings(self, field_value): - if (isinstance(field_value, ast.AST) and - hasattr(node, "targets") and - isinstance(node.targets, list)): - for target in node.targets: - if (hasattr(target, "id") and - (target.id == "order_time_in_force" or - target.id == "order_types" or - target.id == "unfilledtimeout") and - hasattr(field_value, "keys") and - isinstance(field_value.keys, list)): - for key in field_value.keys: - self.visit(key) - - def visit_Name(self, node): - # if the name is in the mapping, update it - if node.id in _name_mapping: - node.id = _name_mapping[node.id] - return node - - def visit_Import(self, node): - # do not update the names in import statements - return node - - def visit_ImportFrom(self, node): - # do not update the names in import statements - if hasattr(node, "module"): - if node.module == "freqtrade.strategy.hyper": - node.module = "freqtrade.strategy" - return node - - def visit_FunctionDef(self, node): - # if the function name is in the mapping, update it - if node.name in _function_mapping: - node.name = _function_mapping[node.name] - return self.generic_visit(node) - - def visit_Attribute(self, node): - # if the attribute name is 'nr_of_successful_buys', - # update it to 'nr_of_successful_entries' - if isinstance(node.value, ast.Name) and \ - node.value.id == 'trades' and \ - node.attr == 'nr_of_successful_buys': - node.attr = 'nr_of_successful_entries' - return self.generic_visit(node) - - def visit_ClassDef(self, node): - # check if the class is derived from IStrategy - if any(isinstance(base, ast.Name) and - base.id == 'IStrategy' for base in node.bases): - # check if the INTERFACE_VERSION variable exists - has_interface_version = any( - isinstance(child, ast.Assign) and - isinstance(child.targets[0], ast.Name) and - child.targets[0].id == 'INTERFACE_VERSION' - for child in node.body - ) - - # if the INTERFACE_VERSION variable does not exist, add it as the first child - if not has_interface_version: - node.body.insert(0, ast.parse('INTERFACE_VERSION = 3').body[0]) - # otherwise, update its value to 3 - else: - for child in node.body: - if isinstance(child, ast.Assign) and \ - isinstance(child.targets[0], ast.Name) and \ - child.targets[0].id == 'INTERFACE_VERSION': - child.value = ast.parse('3').body[0].value - return self.generic_visit(node) - - def visit_Subscript(self, node): - if isinstance(node.slice, ast.Constant): - if node.slice.value in strategy_updater.rename_dict: - # Replace the slice attributes with the values from rename_dict - node.slice.value = strategy_updater.rename_dict[node.slice.value] - if hasattr(node.slice, "elts"): - for elt in node.slice.elts: - if isinstance(elt, ast.Constant) and \ - elt.value in strategy_updater.rename_dict: - elt.value = strategy_updater.rename_dict[elt.value] - return node - - def visit_Constant(self, node): - # do not update the names in import statements - if node.value in \ - strategy_updater.otif_ot_unfilledtimeout: - node.value = \ - strategy_updater.otif_ot_unfilledtimeout[node.value] - return node - - # use the visitor to update the names and functions in the AST - NameUpdater().visit(node) - - # first fix the comments so it understands "\n" properly inside multi line comments. - ast.fix_missing_locations(node) - ast.increment_lineno(node, n=1) - - # generate the new code from the updated AST - return ast.unparse(node) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py new file mode 100644 index 000000000..b62bc7822 --- /dev/null +++ b/freqtrade/strategy/strategyupdater.py @@ -0,0 +1,225 @@ +import ast +import os +import shutil +from pathlib import Path + +import astor + + +class StrategyUpdater: + name_mapping = { + 'ticker_interval': 'timeframe', + 'buy': 'enter_long', + 'sell': 'exit_long', + 'buy_tag': 'enter_tag', + 'sell_reason': 'exit_reason', + + 'sell_signal': 'exit_signal', + 'custom_sell': 'custom_exit', + 'force_sell': 'force_exit', + 'emergency_sell': 'emergency_exit', + + # Strategy/config settings: + 'use_sell_signal': 'use_exit_signal', + 'sell_profit_only': 'exit_profit_only', + 'sell_profit_offset': 'exit_profit_offset', + 'ignore_roi_if_buy_signal': 'ignore_roi_if_entry_signal', + 'forcebuy_enable': 'force_entry_enable', + } + + function_mapping = { + 'populate_buy_trend': 'populate_entry_trend', + 'populate_sell_trend': 'populate_exit_trend', + 'custom_sell': 'custom_exit', + 'check_buy_timeout': 'check_entry_timeout', + 'check_sell_timeout': 'check_exit_timeout', + # '': '', + } + # order_time_in_force, order_types, unfilledtimeout + otif_ot_unfilledtimeout = { + 'buy': 'entry', + 'sell': 'exit', + } + + # create a dictionary that maps the old column names to the new ones + rename_dict = {'buy': 'enter_long', 'sell': 'exit_long', 'buy_tag': 'enter_tag'} + + def start(self, config, strategy_obj: dict) -> None: + """ + Run strategy updater + It updates a strategy to v3 with the help of the ast-module + :return: None + """ + + source_file = strategy_obj['location'] + strategies_backup_folder = Path.joinpath(config['user_data_dir'], "strategies_orig_updater") + target_file = Path.joinpath(strategies_backup_folder, strategy_obj['location_rel']) + + # read the file + with open(source_file, 'r') as f: + old_code = f.read() + if not os.path.exists(strategies_backup_folder): + os.makedirs(strategies_backup_folder) + + # backup original + # => currently no date after the filename, + # could get overridden pretty fast if this is fired twice! + # The folder is always the same and the file name too (currently). + shutil.copy(source_file, target_file) + + # update the code + new_code = StrategyUpdater.update_code(self, old_code) + + # write the modified code to the destination folder + with open(source_file, 'w') as f: + f.write(new_code) + print(f"conversion of file {source_file} successful.") + + # define the function to update the code + def update_code(self, code): + # parse the code into an AST + tree = ast.parse(code) + + # use the AST to update the code + updated_code = self.modify_ast( + tree) + + # return the modified code without executing it + return updated_code + + # function that uses the ast module to update the code + def modify_ast(self, tree): # noqa + # use the visitor to update the names and functions in the AST + NameUpdater().visit(tree) + + # first fix the comments, so it understands "\n" properly inside multi line comments. + ast.fix_missing_locations(tree) + ast.increment_lineno(tree, n=1) + + # generate the new code from the updated AST + return astor.to_source(tree) + + +# Here we go through each respective node, slice, elt, key ... to replace outdated entries. +class NameUpdater(ast.NodeTransformer): + def generic_visit(self, node): + # traverse the AST recursively by calling the visitor method for each child node + if hasattr(node, "_fields"): + for field_name, field_value in ast.iter_fields(node): + self.visit(field_value) + self.generic_visit(field_value) + self.check_fields(field_value) + self.check_strategy_and_config_settings(node, field_value) + # add this check to handle the case where field_value is a slice + if isinstance(field_value, ast.Slice): + self.visit(field_value) + # add this check to handle the case where field_value is a target + if isinstance(field_value, ast.expr_context): + self.visit(field_value) + + def check_fields(self, field_value): + if isinstance(field_value, list): + for item in field_value: + if isinstance(item, ast.AST): + self.visit(item) + + def check_strategy_and_config_settings(self, node, field_value): + if (isinstance(field_value, ast.AST) and + hasattr(node, "targets") and + isinstance(node.targets, list)): + for target in node.targets: + if (hasattr(target, "id") and + hasattr(field_value, "keys") and + isinstance(field_value.keys, list)): + if (target.id == "order_time_in_force" or + target.id == "order_types" or + target.id == "unfilledtimeout"): + for key in field_value.keys: + self.visit(key) + # Check if the target is a Subscript object with a "value" attribute + if isinstance(target, ast.Subscript) and hasattr(target.value, "attr"): + if target.value.attr == "loc": + self.visit(target) + + def visit_Name(self, node): + # if the name is in the mapping, update it + if node.id in StrategyUpdater.name_mapping: + node.id = StrategyUpdater.name_mapping[node.id] + return node + + def visit_Import(self, node): + # do not update the names in import statements + return node + + def visit_ImportFrom(self, node): + # do not update the names in import statements + if hasattr(node, "module"): + if node.module == "freqtrade.strategy.hyper": + node.module = "freqtrade.strategy" + return node + + def visit_FunctionDef(self, node): + # if the function name is in the mapping, update it + if node.name in StrategyUpdater.function_mapping: + node.name = StrategyUpdater.function_mapping[node.name] + return self.generic_visit(node) + + def visit_Attribute(self, node): + # if the attribute name is 'nr_of_successful_buys', + # update it to 'nr_of_successful_entries' + if isinstance(node.value, ast.Name) and \ + node.value.id == 'trades' and \ + node.attr == 'nr_of_successful_buys': + node.attr = 'nr_of_successful_entries' + return self.generic_visit(node) + + def visit_ClassDef(self, node): + # check if the class is derived from IStrategy + if any(isinstance(base, ast.Name) and + base.id == 'IStrategy' for base in node.bases): + # check if the INTERFACE_VERSION variable exists + has_interface_version = any( + isinstance(child, ast.Assign) and + isinstance(child.targets[0], ast.Name) and + child.targets[0].id == 'INTERFACE_VERSION' + for child in node.body + ) + + # if the INTERFACE_VERSION variable does not exist, add it as the first child + if not has_interface_version: + node.body.insert(0, ast.parse('INTERFACE_VERSION = 3').body[0]) + # otherwise, update its value to 3 + else: + for child in node.body: + if isinstance(child, ast.Assign) and \ + isinstance(child.targets[0], ast.Name) and \ + child.targets[0].id == 'INTERFACE_VERSION': + child.value = ast.parse('3').body[0].value + return self.generic_visit(node) + + def visit_Subscript(self, node): + if isinstance(node.slice, ast.Constant): + if node.slice.value in StrategyUpdater.rename_dict: + # Replace the slice attributes with the values from rename_dict + node.slice.value = StrategyUpdater.rename_dict[node.slice.value] + if hasattr(node.slice, "elts"): + self.visit_slice_elts(node.slice.elts) + if hasattr(node.slice.value, "elts"): + self.visit_slice_elts(node.slice.value.elts) + return node + + # elts can have elts (technically recursively) + def visit_slice_elts(self, elts): + for elt in elts: + if isinstance(elt, ast.Constant) and elt.value in StrategyUpdater.rename_dict: + elt.value = StrategyUpdater.rename_dict[elt.value] + elif hasattr(elt, "elts"): + self.visit_slice_elts(elt.elts) + + def visit_Constant(self, node): + # do not update the names in import statements + if node.value in \ + StrategyUpdater.otif_ot_unfilledtimeout: + node.value = \ + StrategyUpdater.otif_ot_unfilledtimeout[node.value] + return node From 82218d01f48051e2a43ad097a7453f3a1da533ac Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 30 Dec 2022 21:48:06 +0100 Subject: [PATCH 03/30] sped up the function generic_visit that now skips unnecessary fields added mentioning of skipped class names since they could not be found --- freqtrade/commands/strategy_utils_commands.py | 5 +++-- freqtrade/strategy/strategyupdater.py | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 13405089f..cf7ba5e13 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -25,11 +25,12 @@ def start_strategy_update(args: Dict[str, Any]) -> None: config, enum_failed=True, recursive=config.get('recursive_strategy_search', False)) filtered_strategy_objs = [] - for args_strategy in args['strategy_list']: - for strategy_obj in strategy_objs: + for strategy_obj in strategy_objs: + for args_strategy in args['strategy_list']: if strategy_obj['name'] == args_strategy and strategy_obj not in filtered_strategy_objs: filtered_strategy_objs.append(strategy_obj) break + print(f"strategy {strategy_obj['name']} could not be loaded or found and is skipped.") for filtered_strategy_obj in filtered_strategy_objs: # Initialize backtesting object diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index b62bc7822..74bb4454c 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -52,6 +52,7 @@ class StrategyUpdater: """ source_file = strategy_obj['location'] + print(f"started conversion of {source_file}") strategies_backup_folder = Path.joinpath(config['user_data_dir'], "strategies_orig_updater") target_file = Path.joinpath(strategies_backup_folder, strategy_obj['location_rel']) @@ -106,6 +107,8 @@ class NameUpdater(ast.NodeTransformer): # traverse the AST recursively by calling the visitor method for each child node if hasattr(node, "_fields"): for field_name, field_value in ast.iter_fields(node): + if not isinstance(field_value, ast.AST): + continue # to avoid unnecessary loops self.visit(field_value) self.generic_visit(field_value) self.check_fields(field_value) @@ -204,8 +207,9 @@ class NameUpdater(ast.NodeTransformer): node.slice.value = StrategyUpdater.rename_dict[node.slice.value] if hasattr(node.slice, "elts"): self.visit_slice_elts(node.slice.elts) - if hasattr(node.slice.value, "elts"): - self.visit_slice_elts(node.slice.value.elts) + if hasattr(node.slice, "value"): + if hasattr(node.slice.value, "elts"): + self.visit_slice_elts(node.slice.value.elts) return node # elts can have elts (technically recursively) From a51e44eea304e548c74c190aed6a6adb8f81d0ed Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 1 Jan 2023 12:37:15 +0100 Subject: [PATCH 04/30] Adding tests --- freqtrade/commands/strategy_utils_commands.py | 10 +++- freqtrade/strategy/strategyupdater.py | 35 +++++------- tests/test_strategy_updater.py | 56 +++++++++++++++++++ 3 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 tests/test_strategy_updater.py diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index cf7ba5e13..1f3c27e0d 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -25,12 +25,16 @@ def start_strategy_update(args: Dict[str, Any]) -> None: config, enum_failed=True, recursive=config.get('recursive_strategy_search', False)) filtered_strategy_objs = [] - for strategy_obj in strategy_objs: - for args_strategy in args['strategy_list']: + for args_strategy in args['strategy_list']: + found = False + for strategy_obj in strategy_objs: if strategy_obj['name'] == args_strategy and strategy_obj not in filtered_strategy_objs: filtered_strategy_objs.append(strategy_obj) + found = True break - print(f"strategy {strategy_obj['name']} could not be loaded or found and is skipped.") + + if not found: + print(f"strategy {strategy_obj['name']} could not be loaded or found and is skipped.") for filtered_strategy_obj in filtered_strategy_objs: # Initialize backtesting object diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index 74bb4454c..396d57a8a 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -70,7 +70,6 @@ class StrategyUpdater: # update the code new_code = StrategyUpdater.update_code(self, old_code) - # write the modified code to the destination folder with open(source_file, 'w') as f: f.write(new_code) @@ -82,8 +81,7 @@ class StrategyUpdater: tree = ast.parse(code) # use the AST to update the code - updated_code = self.modify_ast( - tree) + updated_code = self.modify_ast(self, tree) # return the modified code without executing it return updated_code @@ -107,18 +105,8 @@ class NameUpdater(ast.NodeTransformer): # traverse the AST recursively by calling the visitor method for each child node if hasattr(node, "_fields"): for field_name, field_value in ast.iter_fields(node): - if not isinstance(field_value, ast.AST): - continue # to avoid unnecessary loops - self.visit(field_value) - self.generic_visit(field_value) - self.check_fields(field_value) self.check_strategy_and_config_settings(node, field_value) - # add this check to handle the case where field_value is a slice - if isinstance(field_value, ast.Slice): - self.visit(field_value) - # add this check to handle the case where field_value is a target - if isinstance(field_value, ast.expr_context): - self.visit(field_value) + self.check_fields(field_value) def check_fields(self, field_value): if isinstance(field_value, list): @@ -139,10 +127,6 @@ class NameUpdater(ast.NodeTransformer): target.id == "unfilledtimeout"): for key in field_value.keys: self.visit(key) - # Check if the target is a Subscript object with a "value" attribute - if isinstance(target, ast.Subscript) and hasattr(target.value, "attr"): - if target.value.attr == "loc": - self.visit(target) def visit_Name(self, node): # if the name is in the mapping, update it @@ -154,11 +138,14 @@ class NameUpdater(ast.NodeTransformer): # do not update the names in import statements return node + # This function is currently never successfully triggered + # since freqtrade currently only allows valid code to be processed. + # The module .hyper does not anymore exist and by that fails to even + # reach this function to be updated currently. def visit_ImportFrom(self, node): - # do not update the names in import statements - if hasattr(node, "module"): - if node.module == "freqtrade.strategy.hyper": - node.module = "freqtrade.strategy" + # if hasattr(node, "module"): + # if node.module == "freqtrade.strategy.hyper": + # node.module = "freqtrade.strategy" return node def visit_FunctionDef(self, node): @@ -210,6 +197,10 @@ class NameUpdater(ast.NodeTransformer): if hasattr(node.slice, "value"): if hasattr(node.slice.value, "elts"): self.visit_slice_elts(node.slice.value.elts) + # Check if the target is a Subscript object with a "value" attribute + # if isinstance(target, ast.Subscript) and hasattr(target.value, "attr"): + # if target.value.attr == "loc": + # self.visit(target) return node # elts can have elts (technically recursively) diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py new file mode 100644 index 000000000..6997abdce --- /dev/null +++ b/tests/test_strategy_updater.py @@ -0,0 +1,56 @@ +# pragma pylint: disable=missing-docstring, protected-access, invalid-name + +from freqtrade.strategy.strategyupdater import StrategyUpdater + + +def test_strategy_updater(default_conf, caplog) -> None: + modified_code1 = StrategyUpdater.update_code(StrategyUpdater, """ +class testClass(IStrategy): + def populate_buy_trend(): + pass + def populate_sell_trend(): + pass + def check_buy_timeout(): + pass + def check_sell_timeout(): + pass + def custom_sell(): + pass +""") + + modified_code2 = StrategyUpdater.update_code(StrategyUpdater, """ +buy_some_parameter = IntParameter(space='buy') +sell_some_parameter = IntParameter(space='sell') +ticker_interval = '15m' +""") + modified_code3 = StrategyUpdater.update_code(StrategyUpdater, """ +use_sell_signal = True +sell_profit_only = True +sell_profit_offset = True +ignore_roi_if_buy_signal = True +forcebuy_enable = True +""") + modified_code4 = StrategyUpdater.update_code(StrategyUpdater, """ +dataframe.loc[reduce(lambda x, y: x & y, conditions), 'buy'] = 1 +dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 +""") + assert "populate_entry_trend" in modified_code1 + assert "populate_exit_trend" in modified_code1 + assert "check_entry_timeout" in modified_code1 + assert "check_exit_timeout" in modified_code1 + assert "custom_exit" in modified_code1 + assert "INTERFACE_VERSION = 3" in modified_code1 + + assert "timeframe" in modified_code2 + # check for not editing hyperopt spaces + assert "space='buy'" in modified_code2 + assert "space='sell'" in modified_code2 + + assert "use_exit_signal" in modified_code3 + assert "exit_profit_only" in modified_code3 + assert "exit_profit_offset" in modified_code3 + assert "ignore_roi_if_entry_signal" in modified_code3 + assert "force_entry_enable" in modified_code3 + + assert "enter_long" in modified_code4 + assert "exit_long" in modified_code4 From 762dd4f024907ea9f9ddd0ac303eaf9f77e6dec4 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 1 Jan 2023 18:57:38 +0100 Subject: [PATCH 05/30] Adding tests added more code inside NameUpdater to grab more variables. --- freqtrade/strategy/strategyupdater.py | 40 +++++++++++++++++++++------ tests/test_strategy_updater.py | 11 +++++++- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index 396d57a8a..5b4bb8be0 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -128,6 +128,15 @@ class NameUpdater(ast.NodeTransformer): for key in field_value.keys: self.visit(key) + def check_args(self, node): + if isinstance(node.args, ast.arguments): + self.check_args(node.args) + if hasattr(node, "args"): + if isinstance(node.args, list): + for arg in node.args: + arg.arg = StrategyUpdater.name_mapping[arg.arg] + return node + def visit_Name(self, node): # if the name is in the mapping, update it if node.id in StrategyUpdater.name_mapping: @@ -152,6 +161,8 @@ class NameUpdater(ast.NodeTransformer): # if the function name is in the mapping, update it if node.name in StrategyUpdater.function_mapping: node.name = StrategyUpdater.function_mapping[node.name] + if hasattr(node, "args"): + self.check_args(node) return self.generic_visit(node) def visit_Attribute(self, node): @@ -193,10 +204,10 @@ class NameUpdater(ast.NodeTransformer): # Replace the slice attributes with the values from rename_dict node.slice.value = StrategyUpdater.rename_dict[node.slice.value] if hasattr(node.slice, "elts"): - self.visit_slice_elts(node.slice.elts) + self.visit_elts(node.slice.elts) if hasattr(node.slice, "value"): if hasattr(node.slice.value, "elts"): - self.visit_slice_elts(node.slice.value.elts) + self.visit_elts(node.slice.value.elts) # Check if the target is a Subscript object with a "value" attribute # if isinstance(target, ast.Subscript) and hasattr(target.value, "attr"): # if target.value.attr == "loc": @@ -204,12 +215,25 @@ class NameUpdater(ast.NodeTransformer): return node # elts can have elts (technically recursively) - def visit_slice_elts(self, elts): - for elt in elts: - if isinstance(elt, ast.Constant) and elt.value in StrategyUpdater.rename_dict: - elt.value = StrategyUpdater.rename_dict[elt.value] - elif hasattr(elt, "elts"): - self.visit_slice_elts(elt.elts) + def visit_elts(self, elts): + if isinstance(elts, list): + for elt in elts: + self.visit_elt(elt) + else: + self.visit_elt(elts) + + # sub function again needed since the structure itself is highly flexible ... + def visit_elt(self, elt): + if isinstance(elt, ast.Constant) and elt.value in StrategyUpdater.rename_dict: + elt.value = StrategyUpdater.rename_dict[elt.value] + if hasattr(elt, "elts"): + self.visit_elts(elt.elts) + if hasattr(elt, "args"): + if isinstance(elt.args, ast.arguments): + self.visit_elts(elt.args) + else: + for arg in elt.args: + self.visit_elts(arg) def visit_Constant(self, node): # do not update the names in import statements diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 6997abdce..cf18fcc25 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -4,6 +4,11 @@ from freqtrade.strategy.strategyupdater import StrategyUpdater def test_strategy_updater(default_conf, caplog) -> None: + modified_code5 = StrategyUpdater.update_code(StrategyUpdater, """ +def confirm_trade_exit(sell_reason: str): + pass +""") + modified_code1 = StrategyUpdater.update_code(StrategyUpdater, """ class testClass(IStrategy): def populate_buy_trend(): @@ -31,9 +36,10 @@ ignore_roi_if_buy_signal = True forcebuy_enable = True """) modified_code4 = StrategyUpdater.update_code(StrategyUpdater, """ -dataframe.loc[reduce(lambda x, y: x & y, conditions), 'buy'] = 1 +dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 """) + assert "populate_entry_trend" in modified_code1 assert "populate_exit_trend" in modified_code1 assert "check_entry_timeout" in modified_code1 @@ -54,3 +60,6 @@ dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 assert "enter_long" in modified_code4 assert "exit_long" in modified_code4 + assert "enter_tag" in modified_code4 + + assert "exit_reason" in modified_code5 From 66f7c913570b29be29d180caa1e50ff75f2438b2 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 1 Jan 2023 22:03:45 +0100 Subject: [PATCH 06/30] Adding tests added more code inside NameUpdater to grab more variables. --- freqtrade/strategy/strategyupdater.py | 18 +++++++++- requirements.txt | 20 +++++++++++ tests/test_strategy_updater.py | 50 +++++++++++++++++++++------ 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index 5b4bb8be0..1a0423076 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -107,12 +107,16 @@ class NameUpdater(ast.NodeTransformer): for field_name, field_value in ast.iter_fields(node): self.check_strategy_and_config_settings(node, field_value) self.check_fields(field_value) + for child in ast.iter_child_nodes(node): + self.generic_visit(child) def check_fields(self, field_value): if isinstance(field_value, list): for item in field_value: - if isinstance(item, ast.AST): + if isinstance(item, ast.AST) or isinstance(item, ast.If): self.visit(item) + if isinstance(field_value, ast.Name): + self.visit_Name(field_value) def check_strategy_and_config_settings(self, node, field_value): if (isinstance(field_value, ast.AST) and @@ -157,6 +161,11 @@ class NameUpdater(ast.NodeTransformer): # node.module = "freqtrade.strategy" return node + def visit_If(self, node: ast.If): + for child in ast.iter_child_nodes(node): + self.visit(child) + return self.generic_visit(node) + def visit_FunctionDef(self, node): # if the function name is in the mapping, update it if node.name in StrategyUpdater.function_mapping: @@ -165,6 +174,13 @@ class NameUpdater(ast.NodeTransformer): self.check_args(node) return self.generic_visit(node) + def visit_Assign(self, node): + if hasattr(node, "targets") and isinstance(node.targets, list): + for target in node.targets: + if hasattr(target, "id") and target.id in StrategyUpdater.name_mapping: + target.id = StrategyUpdater.name_mapping[target.id] + return node + def visit_Attribute(self, node): # if the attribute name is 'nr_of_successful_buys', # update it to 'nr_of_successful_entries' diff --git a/requirements.txt b/requirements.txt index 90bc4f702..3229ec3c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,3 +57,23 @@ schedule==1.1.0 #WS Messages websockets==10.4 janus==1.0.0 + +pytest~=7.2.0 +freqtrade~=2022.12.dev0 +filelock~=3.8.2 +plotly~=5.11.0 +setuptools~=65.6.3 +starlette~=0.22.0 +gym~=0.21.0 +torch~=1.13.1 +scikit-learn~=1.1.3 +scipy~=1.9.3 +xgboost~=1.7.2 +catboost~=1.1.1 +lightgbm~=3.3.3 +astor~=0.8.1 +ta~=0.10.2 +finta~=1.3 +tapy~=1.9.1 +matplotlib~=3.6.2 +PyYAML~=6.0 diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index cf18fcc25..5736e5c76 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -4,11 +4,11 @@ from freqtrade.strategy.strategyupdater import StrategyUpdater def test_strategy_updater(default_conf, caplog) -> None: - modified_code5 = StrategyUpdater.update_code(StrategyUpdater, """ -def confirm_trade_exit(sell_reason: str): - pass + modified_code2 = StrategyUpdater.update_code(StrategyUpdater, """ +ticker_interval = '15m' +buy_some_parameter = IntParameter(space='buy') +sell_some_parameter = IntParameter(space='sell') """) - modified_code1 = StrategyUpdater.update_code(StrategyUpdater, """ class testClass(IStrategy): def populate_buy_trend(): @@ -21,12 +21,6 @@ class testClass(IStrategy): pass def custom_sell(): pass -""") - - modified_code2 = StrategyUpdater.update_code(StrategyUpdater, """ -buy_some_parameter = IntParameter(space='buy') -sell_some_parameter = IntParameter(space='sell') -ticker_interval = '15m' """) modified_code3 = StrategyUpdater.update_code(StrategyUpdater, """ use_sell_signal = True @@ -38,6 +32,32 @@ forcebuy_enable = True modified_code4 = StrategyUpdater.update_code(StrategyUpdater, """ dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 +""") + modified_code5 = StrategyUpdater.update_code(StrategyUpdater, """ +def confirm_trade_exit(sell_reason: str): + pass + """) + modified_code6 = StrategyUpdater.update_code(StrategyUpdater, """ +order_time_in_force = { + 'buy': 'gtc', + 'sell': 'ioc' +} +order_types = { + 'buy': 'limit', + 'sell': 'market', + 'stoploss': 'market', + 'stoploss_on_exchange': False +} +unfilledtimeout = { + 'buy': 1, + 'sell': 2 +} +""") + + modified_code7 = StrategyUpdater.update_code(StrategyUpdater, """ +def confirm_trade_exit(sell_reason): + if (sell_reason == 'stop_loss'): + pass """) assert "populate_entry_trend" in modified_code1 @@ -63,3 +83,13 @@ dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 assert "enter_tag" in modified_code4 assert "exit_reason" in modified_code5 + + assert "'entry': 'gtc'" in modified_code6 + assert "'exit': 'ioc'" in modified_code6 + assert "'entry': 'limit'" in modified_code6 + assert "'exit': 'market'" in modified_code6 + assert "'entry': 1" in modified_code6 + assert "'exit': 2" in modified_code6 + + assert "exit_reason" in modified_code7 + assert "exit_reason == 'stop_loss'" in modified_code7 From e89609dc3a9fc32003243785c9a2e173a4ebd09d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 2 Jan 2023 08:51:54 +0100 Subject: [PATCH 07/30] Fix crash due to invalid parameter --- freqtrade/strategy/strategyupdater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index 1a0423076..ab757e0a9 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -81,7 +81,7 @@ class StrategyUpdater: tree = ast.parse(code) # use the AST to update the code - updated_code = self.modify_ast(self, tree) + updated_code = self.modify_ast(tree) # return the modified code without executing it return updated_code From a712c5d42c567751be0949a2202f48d138547c25 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 2 Jan 2023 08:44:00 +0100 Subject: [PATCH 08/30] Improve if formatting --- freqtrade/strategy/strategyupdater.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index ab757e0a9..a0547932c 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -184,9 +184,11 @@ class NameUpdater(ast.NodeTransformer): def visit_Attribute(self, node): # if the attribute name is 'nr_of_successful_buys', # update it to 'nr_of_successful_entries' - if isinstance(node.value, ast.Name) and \ - node.value.id == 'trades' and \ - node.attr == 'nr_of_successful_buys': + if ( + isinstance(node.value, ast.Name) + and node.value.id == 'trades' + and node.attr == 'nr_of_successful_buys' + ): node.attr = 'nr_of_successful_entries' return self.generic_visit(node) @@ -208,9 +210,11 @@ class NameUpdater(ast.NodeTransformer): # otherwise, update its value to 3 else: for child in node.body: - if isinstance(child, ast.Assign) and \ - isinstance(child.targets[0], ast.Name) and \ - child.targets[0].id == 'INTERFACE_VERSION': + if ( + isinstance(child, ast.Assign) + and isinstance(child.targets[0], ast.Name) + and child.targets[0].id == 'INTERFACE_VERSION' + ): child.value = ast.parse('3').body[0].value return self.generic_visit(node) @@ -253,8 +257,6 @@ class NameUpdater(ast.NodeTransformer): def visit_Constant(self, node): # do not update the names in import statements - if node.value in \ - StrategyUpdater.otif_ot_unfilledtimeout: - node.value = \ - StrategyUpdater.otif_ot_unfilledtimeout[node.value] + if node.value in StrategyUpdater.otif_ot_unfilledtimeout: + node.value = StrategyUpdater.otif_ot_unfilledtimeout[node.value] return node From df25dbc048594870d6e5c9356575b47326d8ddef Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 2 Jan 2023 08:52:18 +0100 Subject: [PATCH 09/30] Don't require a configuration for strategy-updater --- freqtrade/commands/arguments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9990ad230..a3cdc378a 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -111,7 +111,8 @@ ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-freqaimodels", "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter", - "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"] + "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv", + "strategy-updater"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] From 61d7129d7ce65edcbbdd75f7047d47ad75051cce Mon Sep 17 00:00:00 2001 From: hippocritical <41228167+hippocritical@users.noreply.github.com> Date: Mon, 2 Jan 2023 16:51:05 +0100 Subject: [PATCH 10/30] Update freqtrade/commands/strategy_utils_commands.py Co-authored-by: Matthias --- freqtrade/commands/strategy_utils_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 1f3c27e0d..0325f411c 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -39,4 +39,4 @@ def start_strategy_update(args: Dict[str, Any]) -> None: for filtered_strategy_obj in filtered_strategy_objs: # Initialize backtesting object instance_strategy_updater = StrategyUpdater() - StrategyUpdater.start(instance_strategy_updater, config, filtered_strategy_obj) + self.start(config, filtered_strategy_obj) From 0817e1698f654a13fd6f01c2519dee5f1cc19f99 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Mon, 2 Jan 2023 20:45:56 +0100 Subject: [PATCH 11/30] requirements thinned out again StrategyResolver.search_all_objects(enum_failed) set to False since we got no use in True shortened update_code call added modified_code8 test which currently still fails. (and thereby is commented out) --- freqtrade/commands/strategy_utils_commands.py | 2 +- freqtrade/strategy/strategyupdater.py | 8 +++++--- requirements.txt | 18 ------------------ tests/test_strategy_updater.py | 11 +++++++++++ 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 1f3c27e0d..5d8ede9a2 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -22,7 +22,7 @@ def start_strategy_update(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) strategy_objs = StrategyResolver.search_all_objects( - config, enum_failed=True, recursive=config.get('recursive_strategy_search', False)) + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) filtered_strategy_objs = [] for args_strategy in args['strategy_list']: diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index 1a0423076..7f3a1feff 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -69,7 +69,7 @@ class StrategyUpdater: shutil.copy(source_file, target_file) # update the code - new_code = StrategyUpdater.update_code(self, old_code) + new_code = self.update_code(old_code) # write the modified code to the destination folder with open(source_file, 'w') as f: f.write(new_code) @@ -113,7 +113,8 @@ class NameUpdater(ast.NodeTransformer): def check_fields(self, field_value): if isinstance(field_value, list): for item in field_value: - if isinstance(item, ast.AST) or isinstance(item, ast.If): + if (isinstance(item, ast.AST) or isinstance(item, ast.If) or + isinstance(item, ast.Expr)): self.visit(item) if isinstance(field_value, ast.Name): self.visit_Name(field_value) @@ -138,7 +139,8 @@ class NameUpdater(ast.NodeTransformer): if hasattr(node, "args"): if isinstance(node.args, list): for arg in node.args: - arg.arg = StrategyUpdater.name_mapping[arg.arg] + if arg.arg in StrategyUpdater.name_mapping: + arg.arg = StrategyUpdater.name_mapping[arg.arg] return node def visit_Name(self, node): diff --git a/requirements.txt b/requirements.txt index 3229ec3c4..da1db316b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,22 +58,4 @@ schedule==1.1.0 websockets==10.4 janus==1.0.0 -pytest~=7.2.0 -freqtrade~=2022.12.dev0 -filelock~=3.8.2 -plotly~=5.11.0 -setuptools~=65.6.3 -starlette~=0.22.0 -gym~=0.21.0 -torch~=1.13.1 -scikit-learn~=1.1.3 -scipy~=1.9.3 -xgboost~=1.7.2 -catboost~=1.1.1 -lightgbm~=3.3.3 astor~=0.8.1 -ta~=0.10.2 -finta~=1.3 -tapy~=1.9.1 -matplotlib~=3.6.2 -PyYAML~=6.0 diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 5736e5c76..682c715fe 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -59,6 +59,11 @@ def confirm_trade_exit(sell_reason): if (sell_reason == 'stop_loss'): pass """) + # modified_code8 = StrategyUpdater.update_code(StrategyUpdater, """ + # sell_reason == 'sell_signal' + # sell_reason == 'force_sell' + # sell_reason == 'emergency_sell' + # """) assert "populate_entry_trend" in modified_code1 assert "populate_exit_trend" in modified_code1 @@ -93,3 +98,9 @@ def confirm_trade_exit(sell_reason): assert "exit_reason" in modified_code7 assert "exit_reason == 'stop_loss'" in modified_code7 + + # those tests currently don't work, next in line. + # assert "exit_signal" in modified_code8 + # assert "exit_reason" in modified_code8 + # assert "force_exit" in modified_code8 + # assert "emergency_exit" in modified_code8 From 71ec32ac9e8360a87b9aeec32327f2824af24f0f Mon Sep 17 00:00:00 2001 From: hippocritical Date: Mon, 2 Jan 2023 23:35:51 +0100 Subject: [PATCH 12/30] removed prints for strategy could not be loaded changed back to ast, astor is not really needed. --- freqtrade/commands/strategy_utils_commands.py | 8 +------- freqtrade/strategy/strategyupdater.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 5aeac2266..4a7eacda0 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -26,17 +26,11 @@ def start_strategy_update(args: Dict[str, Any]) -> None: filtered_strategy_objs = [] for args_strategy in args['strategy_list']: - found = False for strategy_obj in strategy_objs: if strategy_obj['name'] == args_strategy and strategy_obj not in filtered_strategy_objs: filtered_strategy_objs.append(strategy_obj) - found = True break - if not found: - print(f"strategy {strategy_obj['name']} could not be loaded or found and is skipped.") - for filtered_strategy_obj in filtered_strategy_objs: - # Initialize backtesting object instance_strategy_updater = StrategyUpdater() - self.start(config, filtered_strategy_obj) + instance_strategy_updater.start(config, filtered_strategy_obj) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index 8a55de1b6..d12d3eaaa 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -3,8 +3,6 @@ import os import shutil from pathlib import Path -import astor - class StrategyUpdater: name_mapping = { @@ -59,7 +57,7 @@ class StrategyUpdater: # read the file with open(source_file, 'r') as f: old_code = f.read() - if not os.path.exists(strategies_backup_folder): + if not strategies_backup_folder.is_dir(): os.makedirs(strategies_backup_folder) # backup original @@ -96,7 +94,7 @@ class StrategyUpdater: ast.increment_lineno(tree, n=1) # generate the new code from the updated AST - return astor.to_source(tree) + return ast.dump(tree) # Here we go through each respective node, slice, elt, key ... to replace outdated entries. @@ -187,9 +185,9 @@ class NameUpdater(ast.NodeTransformer): # if the attribute name is 'nr_of_successful_buys', # update it to 'nr_of_successful_entries' if ( - isinstance(node.value, ast.Name) - and node.value.id == 'trades' - and node.attr == 'nr_of_successful_buys' + isinstance(node.value, ast.Name) + and node.value.id == 'trades' + and node.attr == 'nr_of_successful_buys' ): node.attr = 'nr_of_successful_entries' return self.generic_visit(node) @@ -213,9 +211,9 @@ class NameUpdater(ast.NodeTransformer): else: for child in node.body: if ( - isinstance(child, ast.Assign) - and isinstance(child.targets[0], ast.Name) - and child.targets[0].id == 'INTERFACE_VERSION' + isinstance(child, ast.Assign) + and isinstance(child.targets[0], ast.Name) + and child.targets[0].id == 'INTERFACE_VERSION' ): child.value = ast.parse('3').body[0].value return self.generic_visit(node) From ed55296d202d725b5d2d5e95e60273a8d21dcf1a Mon Sep 17 00:00:00 2001 From: hippocritical Date: Wed, 4 Jan 2023 23:49:33 +0100 Subject: [PATCH 13/30] removed prints for strategy could not be loaded Changed logic to contain much less if conditions currently still missing: Webhook terminology, Telegram notification settings, Strategy/Config settings --- freqtrade/strategy/strategyupdater.py | 124 ++++++++++++-------------- tests/test_strategy_updater.py | 31 ++++--- 2 files changed, 73 insertions(+), 82 deletions(-) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index d12d3eaaa..ad14bb903 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -3,6 +3,8 @@ import os import shutil from pathlib import Path +import astor + class StrategyUpdater: name_mapping = { @@ -79,7 +81,7 @@ class StrategyUpdater: tree = ast.parse(code) # use the AST to update the code - updated_code = self.modify_ast(tree) + updated_code = self.modify_ast(self, tree) # return the modified code without executing it return updated_code @@ -94,67 +96,67 @@ class StrategyUpdater: ast.increment_lineno(tree, n=1) # generate the new code from the updated AST - return ast.dump(tree) + return astor.to_source(tree) # Here we go through each respective node, slice, elt, key ... to replace outdated entries. class NameUpdater(ast.NodeTransformer): def generic_visit(self, node): - # traverse the AST recursively by calling the visitor method for each child node - if hasattr(node, "_fields"): - for field_name, field_value in ast.iter_fields(node): - self.check_strategy_and_config_settings(node, field_value) - self.check_fields(field_value) - for child in ast.iter_child_nodes(node): - self.generic_visit(child) - def check_fields(self, field_value): - if isinstance(field_value, list): - for item in field_value: - if (isinstance(item, ast.AST) or isinstance(item, ast.If) or - isinstance(item, ast.Expr)): - self.visit(item) - if isinstance(field_value, ast.Name): - self.visit_Name(field_value) + # space is not yet transferred from buy/sell to entry/exit and thereby has to be skipped. + if isinstance(node, ast.keyword): + if node.arg == "space": + return node - def check_strategy_and_config_settings(self, node, field_value): - if (isinstance(field_value, ast.AST) and - hasattr(node, "targets") and - isinstance(node.targets, list)): - for target in node.targets: - if (hasattr(target, "id") and - hasattr(field_value, "keys") and - isinstance(field_value.keys, list)): - if (target.id == "order_time_in_force" or - target.id == "order_types" or - target.id == "unfilledtimeout"): - for key in field_value.keys: - self.visit(key) + # from here on this is the original function. + for field, old_value in ast.iter_fields(node): + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, ast.AST): + value = self.visit(value) + if value is None: + continue + elif not isinstance(value, ast.AST): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, ast.AST): + new_node = self.visit(old_value) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node - def check_args(self, node): - if isinstance(node.args, ast.arguments): - self.check_args(node.args) - if hasattr(node, "args"): - if isinstance(node.args, list): - for arg in node.args: - if arg.arg in StrategyUpdater.name_mapping: - arg.arg = StrategyUpdater.name_mapping[arg.arg] + def visit_Expr(self, node): + node.value.left.id = self.check_dict(StrategyUpdater.name_mapping, node.value.left.id) + self.visit(node.value) + return node + + # Renames an element if contained inside a dictionary. + @staticmethod + def check_dict(current_dict: dict, element: str): + if element in current_dict: + element = current_dict[element] + return element + + def visit_arguments(self, node): + if isinstance(node.args, list): + for arg in node.args: + arg.arg = self.check_dict(StrategyUpdater.name_mapping, arg.arg) return node def visit_Name(self, node): # if the name is in the mapping, update it - if node.id in StrategyUpdater.name_mapping: - node.id = StrategyUpdater.name_mapping[node.id] + node.id = self.check_dict(StrategyUpdater.name_mapping, node.id) return node def visit_Import(self, node): # do not update the names in import statements return node - # This function is currently never successfully triggered - # since freqtrade currently only allows valid code to be processed. - # The module .hyper does not anymore exist and by that fails to even - # reach this function to be updated currently. def visit_ImportFrom(self, node): # if hasattr(node, "module"): # if node.module == "freqtrade.strategy.hyper": @@ -164,33 +166,21 @@ class NameUpdater(ast.NodeTransformer): def visit_If(self, node: ast.If): for child in ast.iter_child_nodes(node): self.visit(child) - return self.generic_visit(node) + return node def visit_FunctionDef(self, node): - # if the function name is in the mapping, update it - if node.name in StrategyUpdater.function_mapping: - node.name = StrategyUpdater.function_mapping[node.name] - if hasattr(node, "args"): - self.check_args(node) - return self.generic_visit(node) - - def visit_Assign(self, node): - if hasattr(node, "targets") and isinstance(node.targets, list): - for target in node.targets: - if hasattr(target, "id") and target.id in StrategyUpdater.name_mapping: - target.id = StrategyUpdater.name_mapping[target.id] + node.name = self.check_dict(StrategyUpdater.function_mapping, node.name) + self.generic_visit(node) return node def visit_Attribute(self, node): - # if the attribute name is 'nr_of_successful_buys', - # update it to 'nr_of_successful_entries' if ( isinstance(node.value, ast.Name) and node.value.id == 'trades' and node.attr == 'nr_of_successful_buys' ): node.attr = 'nr_of_successful_entries' - return self.generic_visit(node) + return node def visit_ClassDef(self, node): # check if the class is derived from IStrategy @@ -216,7 +206,8 @@ class NameUpdater(ast.NodeTransformer): and child.targets[0].id == 'INTERFACE_VERSION' ): child.value = ast.parse('3').body[0].value - return self.generic_visit(node) + self.generic_visit(node) + return node def visit_Subscript(self, node): if isinstance(node.slice, ast.Constant): @@ -228,10 +219,6 @@ class NameUpdater(ast.NodeTransformer): if hasattr(node.slice, "value"): if hasattr(node.slice.value, "elts"): self.visit_elts(node.slice.value.elts) - # Check if the target is a Subscript object with a "value" attribute - # if isinstance(target, ast.Subscript) and hasattr(target.value, "attr"): - # if target.value.attr == "loc": - # self.visit(target) return node # elts can have elts (technically recursively) @@ -241,6 +228,7 @@ class NameUpdater(ast.NodeTransformer): self.visit_elt(elt) else: self.visit_elt(elts) + return elts # sub function again needed since the structure itself is highly flexible ... def visit_elt(self, elt): @@ -254,9 +242,9 @@ class NameUpdater(ast.NodeTransformer): else: for arg in elt.args: self.visit_elts(arg) + return elt def visit_Constant(self, node): - # do not update the names in import statements - if node.value in StrategyUpdater.otif_ot_unfilledtimeout: - node.value = StrategyUpdater.otif_ot_unfilledtimeout[node.value] + node.value = self.check_dict(StrategyUpdater.otif_ot_unfilledtimeout, node.value) + node.value = self.check_dict(StrategyUpdater.name_mapping, node.value) return node diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 682c715fe..3ece2b3d8 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -4,11 +4,6 @@ from freqtrade.strategy.strategyupdater import StrategyUpdater def test_strategy_updater(default_conf, caplog) -> None: - modified_code2 = StrategyUpdater.update_code(StrategyUpdater, """ -ticker_interval = '15m' -buy_some_parameter = IntParameter(space='buy') -sell_some_parameter = IntParameter(space='sell') -""") modified_code1 = StrategyUpdater.update_code(StrategyUpdater, """ class testClass(IStrategy): def populate_buy_trend(): @@ -21,6 +16,11 @@ class testClass(IStrategy): pass def custom_sell(): pass +""") + modified_code2 = StrategyUpdater.update_code(StrategyUpdater, """ +ticker_interval = '15m' +buy_some_parameter = IntParameter(space='buy') +sell_some_parameter = IntParameter(space='sell') """) modified_code3 = StrategyUpdater.update_code(StrategyUpdater, """ use_sell_signal = True @@ -59,11 +59,14 @@ def confirm_trade_exit(sell_reason): if (sell_reason == 'stop_loss'): pass """) - # modified_code8 = StrategyUpdater.update_code(StrategyUpdater, """ - # sell_reason == 'sell_signal' - # sell_reason == 'force_sell' - # sell_reason == 'emergency_sell' - # """) + modified_code8 = StrategyUpdater.update_code(StrategyUpdater, """ +sell_reason == 'sell_signal' +sell_reason == 'force_sell' +sell_reason == 'emergency_sell' +""") + + # currently still missing: + # Webhook terminology, Telegram notification settings, Strategy/Config settings assert "populate_entry_trend" in modified_code1 assert "populate_exit_trend" in modified_code1 @@ -100,7 +103,7 @@ def confirm_trade_exit(sell_reason): assert "exit_reason == 'stop_loss'" in modified_code7 # those tests currently don't work, next in line. - # assert "exit_signal" in modified_code8 - # assert "exit_reason" in modified_code8 - # assert "force_exit" in modified_code8 - # assert "emergency_exit" in modified_code8 + assert "exit_signal" in modified_code8 + assert "exit_reason" in modified_code8 + assert "force_exit" in modified_code8 + assert "emergency_exit" in modified_code8 From 4435c4fd0d1de2e6b57375051d8007ca7435c5ac Mon Sep 17 00:00:00 2001 From: hippocritical Date: Thu, 5 Jan 2023 22:56:06 +0100 Subject: [PATCH 14/30] removed prints for strategy could not be loaded Changed logic to contain much less if conditions currently still missing: Webhook terminology, Telegram notification settings, Strategy/Config settings --- freqtrade/commands/strategy_utils_commands.py | 4 ++++ freqtrade/strategy/strategyupdater.py | 10 +++++----- tests/test_strategy_updater.py | 17 +++++++++-------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 4a7eacda0..75b8a9cc0 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -1,4 +1,5 @@ import logging +import time from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration @@ -33,4 +34,7 @@ def start_strategy_update(args: Dict[str, Any]) -> None: for filtered_strategy_obj in filtered_strategy_objs: instance_strategy_updater = StrategyUpdater() + start = time.perf_counter() instance_strategy_updater.start(config, filtered_strategy_obj) + elapsed = time.perf_counter() - start + print(f"Conversion of {filtered_strategy_obj['name']} took {elapsed:.1f} seconds.") diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index ad14bb903..e26dd5e79 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -52,7 +52,6 @@ class StrategyUpdater: """ source_file = strategy_obj['location'] - print(f"started conversion of {source_file}") strategies_backup_folder = Path.joinpath(config['user_data_dir'], "strategies_orig_updater") target_file = Path.joinpath(strategies_backup_folder, strategy_obj['location_rel']) @@ -73,7 +72,6 @@ class StrategyUpdater: # write the modified code to the destination folder with open(source_file, 'w') as f: f.write(new_code) - print(f"conversion of file {source_file} successful.") # define the function to update the code def update_code(self, code): @@ -81,7 +79,7 @@ class StrategyUpdater: tree = ast.parse(code) # use the AST to update the code - updated_code = self.modify_ast(self, tree) + updated_code = self.modify_ast(tree) # return the modified code without executing it return updated_code @@ -96,6 +94,7 @@ class StrategyUpdater: ast.increment_lineno(tree, n=1) # generate the new code from the updated AST + # without indent {} parameters would just be written straight one after the other. return astor.to_source(tree) @@ -131,8 +130,9 @@ class NameUpdater(ast.NodeTransformer): return node def visit_Expr(self, node): - node.value.left.id = self.check_dict(StrategyUpdater.name_mapping, node.value.left.id) - self.visit(node.value) + if hasattr(node.value, "left") and hasattr(node.value.left, "id"): + node.value.left.id = self.check_dict(StrategyUpdater.name_mapping, node.value.left.id) + self.visit(node.value) return node # Renames an element if contained inside a dictionary. diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 3ece2b3d8..a00340427 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -4,7 +4,8 @@ from freqtrade.strategy.strategyupdater import StrategyUpdater def test_strategy_updater(default_conf, caplog) -> None: - modified_code1 = StrategyUpdater.update_code(StrategyUpdater, """ + instance_strategy_updater = StrategyUpdater() + modified_code1 = instance_strategy_updater.update_code(""" class testClass(IStrategy): def populate_buy_trend(): pass @@ -17,27 +18,27 @@ class testClass(IStrategy): def custom_sell(): pass """) - modified_code2 = StrategyUpdater.update_code(StrategyUpdater, """ + modified_code2 = instance_strategy_updater.update_code(""" ticker_interval = '15m' buy_some_parameter = IntParameter(space='buy') sell_some_parameter = IntParameter(space='sell') """) - modified_code3 = StrategyUpdater.update_code(StrategyUpdater, """ + modified_code3 = instance_strategy_updater.update_code(""" use_sell_signal = True sell_profit_only = True sell_profit_offset = True ignore_roi_if_buy_signal = True forcebuy_enable = True """) - modified_code4 = StrategyUpdater.update_code(StrategyUpdater, """ + modified_code4 = instance_strategy_updater.update_code(""" dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 """) - modified_code5 = StrategyUpdater.update_code(StrategyUpdater, """ + modified_code5 = instance_strategy_updater.update_code(""" def confirm_trade_exit(sell_reason: str): pass """) - modified_code6 = StrategyUpdater.update_code(StrategyUpdater, """ + modified_code6 = instance_strategy_updater.update_code(""" order_time_in_force = { 'buy': 'gtc', 'sell': 'ioc' @@ -54,12 +55,12 @@ unfilledtimeout = { } """) - modified_code7 = StrategyUpdater.update_code(StrategyUpdater, """ + modified_code7 = instance_strategy_updater.update_code(""" def confirm_trade_exit(sell_reason): if (sell_reason == 'stop_loss'): pass """) - modified_code8 = StrategyUpdater.update_code(StrategyUpdater, """ + modified_code8 = instance_strategy_updater.update_code(""" sell_reason == 'sell_signal' sell_reason == 'force_sell' sell_reason == 'emergency_sell' From 06edc5c04491b15b2b43c0b9f0900cbf91958e8d Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 17 Feb 2023 21:01:09 +0100 Subject: [PATCH 15/30] changed to ast_comments, added tests for comments. --- freqtrade/commands/strategy_utils_commands.py | 43 ++++++++++----- freqtrade/strategy/strategyupdater.py | 54 ++++++++++--------- tests/test_strategy_updater.py | 20 +++++++ 3 files changed, 78 insertions(+), 39 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 75b8a9cc0..dc94f2b67 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -1,10 +1,12 @@ import logging +import os import time from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.resolvers import StrategyResolver +from freqtrade.strategy.strategyupdater import StrategyUpdater logger = logging.getLogger(__name__) @@ -17,24 +19,37 @@ def start_strategy_update(args: Dict[str, Any]) -> None: :return: None """ - # Import here to avoid loading backtesting module when it's not used - from freqtrade.strategy.strategyupdater import StrategyUpdater - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) strategy_objs = StrategyResolver.search_all_objects( config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) filtered_strategy_objs = [] - for args_strategy in args['strategy_list']: - for strategy_obj in strategy_objs: - if strategy_obj['name'] == args_strategy and strategy_obj not in filtered_strategy_objs: - filtered_strategy_objs.append(strategy_obj) - break + if hasattr(args, "strategy_list"): + for args_strategy in args['strategy_list']: + for strategy_obj in strategy_objs: + if (strategy_obj['name'] == args_strategy + and strategy_obj not in filtered_strategy_objs): + filtered_strategy_objs.append(strategy_obj) + break - for filtered_strategy_obj in filtered_strategy_objs: - instance_strategy_updater = StrategyUpdater() - start = time.perf_counter() - instance_strategy_updater.start(config, filtered_strategy_obj) - elapsed = time.perf_counter() - start - print(f"Conversion of {filtered_strategy_obj['name']} took {elapsed:.1f} seconds.") + for filtered_strategy_obj in filtered_strategy_objs: + start_conversion(filtered_strategy_obj, config) + else: + processed_locations = set() + for strategy_obj in strategy_objs: + if strategy_obj['location'] not in processed_locations: + processed_locations.add(strategy_obj['location']) + start_conversion(strategy_obj, config) + + +def start_conversion(strategy_obj, config): + # try: + print(f"Conversion of {os.path.basename(strategy_obj['location'])} started.") + instance_strategy_updater = StrategyUpdater() + start = time.perf_counter() + instance_strategy_updater.start(config, strategy_obj) + elapsed = time.perf_counter() - start + print(f"Conversion of {os.path.basename(strategy_obj['location'])} took {elapsed:.1f} seconds.") + # except: + # pass diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index e26dd5e79..db19d4fba 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -1,9 +1,8 @@ -import ast import os import shutil from pathlib import Path -import astor +import ast_comments class StrategyUpdater: @@ -76,7 +75,7 @@ class StrategyUpdater: # define the function to update the code def update_code(self, code): # parse the code into an AST - tree = ast.parse(code) + tree = ast_comments.parse(code) # use the AST to update the code updated_code = self.modify_ast(tree) @@ -90,38 +89,43 @@ class StrategyUpdater: NameUpdater().visit(tree) # first fix the comments, so it understands "\n" properly inside multi line comments. - ast.fix_missing_locations(tree) - ast.increment_lineno(tree, n=1) + ast_comments.fix_missing_locations(tree) + ast_comments.increment_lineno(tree, n=1) # generate the new code from the updated AST # without indent {} parameters would just be written straight one after the other. - return astor.to_source(tree) + + # ast_comments would be amazing since this is the only solution that carries over comments, + # but it does currently not have an unparse function, hopefully in the future ... ! + # return ast_comments.unparse(tree) + + return ast_comments.unparse(tree) # Here we go through each respective node, slice, elt, key ... to replace outdated entries. -class NameUpdater(ast.NodeTransformer): +class NameUpdater(ast_comments.NodeTransformer): def generic_visit(self, node): # space is not yet transferred from buy/sell to entry/exit and thereby has to be skipped. - if isinstance(node, ast.keyword): + if isinstance(node, ast_comments.keyword): if node.arg == "space": return node # from here on this is the original function. - for field, old_value in ast.iter_fields(node): + for field, old_value in ast_comments.iter_fields(node): if isinstance(old_value, list): new_values = [] for value in old_value: - if isinstance(value, ast.AST): + if isinstance(value, ast_comments.AST): value = self.visit(value) if value is None: continue - elif not isinstance(value, ast.AST): + elif not isinstance(value, ast_comments.AST): new_values.extend(value) continue new_values.append(value) old_value[:] = new_values - elif isinstance(old_value, ast.AST): + elif isinstance(old_value, ast_comments.AST): new_node = self.visit(old_value) if new_node is None: delattr(node, field) @@ -163,8 +167,8 @@ class NameUpdater(ast.NodeTransformer): # node.module = "freqtrade.strategy" return node - def visit_If(self, node: ast.If): - for child in ast.iter_child_nodes(node): + def visit_If(self, node: ast_comments.If): + for child in ast_comments.iter_child_nodes(node): self.visit(child) return node @@ -175,7 +179,7 @@ class NameUpdater(ast.NodeTransformer): def visit_Attribute(self, node): if ( - isinstance(node.value, ast.Name) + isinstance(node.value, ast_comments.Name) and node.value.id == 'trades' and node.attr == 'nr_of_successful_buys' ): @@ -184,33 +188,33 @@ class NameUpdater(ast.NodeTransformer): def visit_ClassDef(self, node): # check if the class is derived from IStrategy - if any(isinstance(base, ast.Name) and + if any(isinstance(base, ast_comments.Name) and base.id == 'IStrategy' for base in node.bases): # check if the INTERFACE_VERSION variable exists has_interface_version = any( - isinstance(child, ast.Assign) and - isinstance(child.targets[0], ast.Name) and + isinstance(child, ast_comments.Assign) and + isinstance(child.targets[0], ast_comments.Name) and child.targets[0].id == 'INTERFACE_VERSION' for child in node.body ) # if the INTERFACE_VERSION variable does not exist, add it as the first child if not has_interface_version: - node.body.insert(0, ast.parse('INTERFACE_VERSION = 3').body[0]) + node.body.insert(0, ast_comments.parse('INTERFACE_VERSION = 3').body[0]) # otherwise, update its value to 3 else: for child in node.body: if ( - isinstance(child, ast.Assign) - and isinstance(child.targets[0], ast.Name) + isinstance(child, ast_comments.Assign) + and isinstance(child.targets[0], ast_comments.Name) and child.targets[0].id == 'INTERFACE_VERSION' ): - child.value = ast.parse('3').body[0].value + child.value = ast_comments.parse('3').body[0].value self.generic_visit(node) return node def visit_Subscript(self, node): - if isinstance(node.slice, ast.Constant): + if isinstance(node.slice, ast_comments.Constant): if node.slice.value in StrategyUpdater.rename_dict: # Replace the slice attributes with the values from rename_dict node.slice.value = StrategyUpdater.rename_dict[node.slice.value] @@ -232,12 +236,12 @@ class NameUpdater(ast.NodeTransformer): # sub function again needed since the structure itself is highly flexible ... def visit_elt(self, elt): - if isinstance(elt, ast.Constant) and elt.value in StrategyUpdater.rename_dict: + if isinstance(elt, ast_comments.Constant) and elt.value in StrategyUpdater.rename_dict: elt.value = StrategyUpdater.rename_dict[elt.value] if hasattr(elt, "elts"): self.visit_elts(elt.elts) if hasattr(elt, "args"): - if isinstance(elt.args, ast.arguments): + if isinstance(elt.args, ast_comments.arguments): self.visit_elts(elt.args) else: for arg in elt.args: diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index a00340427..927c5e99f 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -65,7 +65,23 @@ sell_reason == 'sell_signal' sell_reason == 'force_sell' sell_reason == 'emergency_sell' """) + modified_code9 = instance_strategy_updater.update_code(""" +# This is the 1st comment +import talib.abstract as ta +# This is the 2nd comment +import freqtrade.vendor.qtpylib.indicators as qtpylib + +class someStrategy(IStrategy): + # This is the 3rd comment + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + "0": 0.50 + } + + # This is the 4th comment + stoploss = -0.1 +""") # currently still missing: # Webhook terminology, Telegram notification settings, Strategy/Config settings @@ -108,3 +124,7 @@ sell_reason == 'emergency_sell' assert "exit_reason" in modified_code8 assert "force_exit" in modified_code8 assert "emergency_exit" in modified_code8 + + assert "This is the 1st comment" in modified_code9 + assert "This is the 2nd comment" in modified_code9 + assert "This is the 3rd comment" in modified_code9 From bcef00edeef1ea1e5611aef6d6d39eb31448195f Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 17 Feb 2023 21:04:26 +0100 Subject: [PATCH 16/30] changed to ast_comments, added tests for comments. --- user_data/strategies/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 user_data/strategies/.gitkeep diff --git a/user_data/strategies/.gitkeep b/user_data/strategies/.gitkeep deleted file mode 100644 index e69de29bb..000000000 From 87b75134016ed748b5d6732a21fc7ca053ca2631 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 3 Mar 2023 18:53:09 +0100 Subject: [PATCH 17/30] fixed --strategy-list moved ast comments to requirements.txt >=1.0.0 (since that is the first version that adds the comments unparsing) --- freqtrade/commands/strategy_utils_commands.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index dc94f2b67..0124b73b6 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -25,7 +25,7 @@ def start_strategy_update(args: Dict[str, Any]) -> None: config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) filtered_strategy_objs = [] - if hasattr(args, "strategy_list"): + if 'strategy_list' in args: for args_strategy in args['strategy_list']: for strategy_obj in strategy_objs: if (strategy_obj['name'] == args_strategy diff --git a/requirements.txt b/requirements.txt index 40bae63b6..14c468da0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,4 +58,4 @@ schedule==1.1.0 websockets==10.4 janus==1.0.0 -astor~=0.8.1 +ast-comments>=1.0.0 From d0d6f53dec489d022fd301375bf4478646b0a41f Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 5 Mar 2023 16:19:26 +0100 Subject: [PATCH 18/30] fixed github formatting errors --- freqtrade/commands/strategy_utils_commands.py | 6 +++--- freqtrade/strategy/strategyupdater.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 0124b73b6..aca368742 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -1,6 +1,6 @@ import logging -import os import time +from pathlib import Path from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration @@ -45,11 +45,11 @@ def start_strategy_update(args: Dict[str, Any]) -> None: def start_conversion(strategy_obj, config): # try: - print(f"Conversion of {os.path.basename(strategy_obj['location'])} started.") + print(f"Conversion of {Path(strategy_obj['location']).name} started.") instance_strategy_updater = StrategyUpdater() start = time.perf_counter() instance_strategy_updater.start(config, strategy_obj) elapsed = time.perf_counter() - start - print(f"Conversion of {os.path.basename(strategy_obj['location'])} took {elapsed:.1f} seconds.") + print(f"Conversion of {Path(strategy_obj['location']).name} took {elapsed:.1f} seconds.") # except: # pass diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index db19d4fba..6fe1f326c 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -1,4 +1,3 @@ -import os import shutil from pathlib import Path @@ -55,10 +54,10 @@ class StrategyUpdater: target_file = Path.joinpath(strategies_backup_folder, strategy_obj['location_rel']) # read the file - with open(source_file, 'r') as f: + with Path(source_file).open('r') as f: old_code = f.read() if not strategies_backup_folder.is_dir(): - os.makedirs(strategies_backup_folder) + Path(strategies_backup_folder).mkdir(parents=True, exist_ok=True) # backup original # => currently no date after the filename, @@ -69,7 +68,7 @@ class StrategyUpdater: # update the code new_code = self.update_code(old_code) # write the modified code to the destination folder - with open(source_file, 'w') as f: + with Path(source_file).open('w') as f: f.write(new_code) # define the function to update the code From b072fae507200321d2a4cb3ce81f27e2eb879d34 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 5 Mar 2023 18:48:32 +0100 Subject: [PATCH 19/30] added strategy-updater compartment inside utils.md --- docs/utils.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/utils.md b/docs/utils.md index 87c7f6aa6..3d6eda3ce 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -947,7 +947,6 @@ Common arguments: --userdir PATH, --user-data-dir PATH Path to userdata directory. ``` - ### Examples Print trades with id 2 and 3 as json @@ -955,3 +954,13 @@ Print trades with id 2 and 3 as json ``` bash freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json ``` + +### Strategy-Updater +Updates a list strategies or all strategies within the strategies folder to be v3 compliant including futures. +If the command runs without --strategy-list then all files inside the strategies folder will be converted. +``` +usage: freqtrade strategy_updater + +optional arguments: + --strategy-list defines a list of strategies that should be converted +``` From 30fd1e742efead96b3ae12fa75b44d1be7231560 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Mar 2023 07:14:54 +0000 Subject: [PATCH 20/30] Add 3.8 block for strategyUpdater --- freqtrade/commands/strategy_utils_commands.py | 4 ++++ tests/test_strategy_updater.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index aca368742..ed4d0bf1a 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -1,4 +1,5 @@ import logging +import sys import time from pathlib import Path from typing import Any, Dict @@ -19,6 +20,9 @@ def start_strategy_update(args: Dict[str, Any]) -> None: :return: None """ + if sys.version_info == (3, 8): # pragma: no cover + sys.exit("Freqtrade strategy updater requires Python version >= 3.9") + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) strategy_objs = StrategyResolver.search_all_objects( diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 927c5e99f..ea971af72 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -1,8 +1,16 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name +import sys + +import pytest + from freqtrade.strategy.strategyupdater import StrategyUpdater +if sys.version_info < (3, 9): + pytest.skip("StrategyUpdater is not compatible with Python 3.8", allow_module_level=True) + + def test_strategy_updater(default_conf, caplog) -> None: instance_strategy_updater = StrategyUpdater() modified_code1 = instance_strategy_updater.update_code(""" From bfc7f48f17c82b759e8a012e372b3ee92017263a Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 10 Mar 2023 08:59:07 +0100 Subject: [PATCH 21/30] added checks for python3.8 or lower since ast_comments.unparse() needs python 3.9 or higher. testing with python 3.8 would make the build fail tests, skipping it there. --- freqtrade/commands/strategy_utils_commands.py | 47 ++-- tests/test_strategy_updater.py | 227 +++++++++--------- 2 files changed, 143 insertions(+), 131 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index aca368742..56a28cf65 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -1,4 +1,5 @@ import logging +import sys import time from pathlib import Path from typing import Any, Dict @@ -19,28 +20,34 @@ def start_strategy_update(args: Dict[str, Any]) -> None: :return: None """ - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + if sys.version_info <= (3, 8): + print("This code requires Python 3.9 or higher. " + "We cannot continue. " + "Please upgrade your python version to use this command.") - strategy_objs = StrategyResolver.search_all_objects( - config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) - - filtered_strategy_objs = [] - if 'strategy_list' in args: - for args_strategy in args['strategy_list']: - for strategy_obj in strategy_objs: - if (strategy_obj['name'] == args_strategy - and strategy_obj not in filtered_strategy_objs): - filtered_strategy_objs.append(strategy_obj) - break - - for filtered_strategy_obj in filtered_strategy_objs: - start_conversion(filtered_strategy_obj, config) else: - processed_locations = set() - for strategy_obj in strategy_objs: - if strategy_obj['location'] not in processed_locations: - processed_locations.add(strategy_obj['location']) - start_conversion(strategy_obj, config) + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + + filtered_strategy_objs = [] + if 'strategy_list' in args: + for args_strategy in args['strategy_list']: + for strategy_obj in strategy_objs: + if (strategy_obj['name'] == args_strategy + and strategy_obj not in filtered_strategy_objs): + filtered_strategy_objs.append(strategy_obj) + break + + for filtered_strategy_obj in filtered_strategy_objs: + start_conversion(filtered_strategy_obj, config) + else: + processed_locations = set() + for strategy_obj in strategy_objs: + if strategy_obj['location'] not in processed_locations: + processed_locations.add(strategy_obj['location']) + start_conversion(strategy_obj, config) def start_conversion(strategy_obj, config): diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 927c5e99f..3831c2ee6 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -1,130 +1,135 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name +import sys + from freqtrade.strategy.strategyupdater import StrategyUpdater def test_strategy_updater(default_conf, caplog) -> None: - instance_strategy_updater = StrategyUpdater() - modified_code1 = instance_strategy_updater.update_code(""" -class testClass(IStrategy): - def populate_buy_trend(): - pass - def populate_sell_trend(): - pass - def check_buy_timeout(): - pass - def check_sell_timeout(): - pass - def custom_sell(): - pass -""") - modified_code2 = instance_strategy_updater.update_code(""" -ticker_interval = '15m' -buy_some_parameter = IntParameter(space='buy') -sell_some_parameter = IntParameter(space='sell') -""") - modified_code3 = instance_strategy_updater.update_code(""" -use_sell_signal = True -sell_profit_only = True -sell_profit_offset = True -ignore_roi_if_buy_signal = True -forcebuy_enable = True -""") - modified_code4 = instance_strategy_updater.update_code(""" -dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") -dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 -""") - modified_code5 = instance_strategy_updater.update_code(""" -def confirm_trade_exit(sell_reason: str): - pass + if sys.version_info <= (3, 8): + print("skipped tests since python version is 3.8 or lower.") + else: + instance_strategy_updater = StrategyUpdater() + modified_code1 = instance_strategy_updater.update_code(""" + class testClass(IStrategy): + def populate_buy_trend(): + pass + def populate_sell_trend(): + pass + def check_buy_timeout(): + pass + def check_sell_timeout(): + pass + def custom_sell(): + pass """) - modified_code6 = instance_strategy_updater.update_code(""" -order_time_in_force = { - 'buy': 'gtc', - 'sell': 'ioc' -} -order_types = { - 'buy': 'limit', - 'sell': 'market', - 'stoploss': 'market', - 'stoploss_on_exchange': False -} -unfilledtimeout = { - 'buy': 1, - 'sell': 2 -} -""") - - modified_code7 = instance_strategy_updater.update_code(""" -def confirm_trade_exit(sell_reason): - if (sell_reason == 'stop_loss'): + modified_code2 = instance_strategy_updater.update_code(""" + ticker_interval = '15m' + buy_some_parameter = IntParameter(space='buy') + sell_some_parameter = IntParameter(space='sell') + """) + modified_code3 = instance_strategy_updater.update_code(""" + use_sell_signal = True + sell_profit_only = True + sell_profit_offset = True + ignore_roi_if_buy_signal = True + forcebuy_enable = True + """) + modified_code4 = instance_strategy_updater.update_code(""" + dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") + dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 + """) + modified_code5 = instance_strategy_updater.update_code(""" + def confirm_trade_exit(sell_reason: str): pass -""") - modified_code8 = instance_strategy_updater.update_code(""" -sell_reason == 'sell_signal' -sell_reason == 'force_sell' -sell_reason == 'emergency_sell' -""") - modified_code9 = instance_strategy_updater.update_code(""" -# This is the 1st comment -import talib.abstract as ta -# This is the 2nd comment -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class someStrategy(IStrategy): - # This is the 3rd comment - # This attribute will be overridden if the config file contains "minimal_roi" - minimal_roi = { - "0": 0.50 + """) + modified_code6 = instance_strategy_updater.update_code(""" + order_time_in_force = { + 'buy': 'gtc', + 'sell': 'ioc' } + order_types = { + 'buy': 'limit', + 'sell': 'market', + 'stoploss': 'market', + 'stoploss_on_exchange': False + } + unfilledtimeout = { + 'buy': 1, + 'sell': 2 + } + """) - # This is the 4th comment - stoploss = -0.1 -""") - # currently still missing: - # Webhook terminology, Telegram notification settings, Strategy/Config settings + modified_code7 = instance_strategy_updater.update_code(""" + def confirm_trade_exit(sell_reason): + if (sell_reason == 'stop_loss'): + pass + """) + modified_code8 = instance_strategy_updater.update_code(""" + sell_reason == 'sell_signal' + sell_reason == 'force_sell' + sell_reason == 'emergency_sell' + """) + modified_code9 = instance_strategy_updater.update_code(""" + # This is the 1st comment + import talib.abstract as ta + # This is the 2nd comment + import freqtrade.vendor.qtpylib.indicators as qtpylib - assert "populate_entry_trend" in modified_code1 - assert "populate_exit_trend" in modified_code1 - assert "check_entry_timeout" in modified_code1 - assert "check_exit_timeout" in modified_code1 - assert "custom_exit" in modified_code1 - assert "INTERFACE_VERSION = 3" in modified_code1 - assert "timeframe" in modified_code2 - # check for not editing hyperopt spaces - assert "space='buy'" in modified_code2 - assert "space='sell'" in modified_code2 + class someStrategy(IStrategy): + # This is the 3rd comment + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + "0": 0.50 + } - assert "use_exit_signal" in modified_code3 - assert "exit_profit_only" in modified_code3 - assert "exit_profit_offset" in modified_code3 - assert "ignore_roi_if_entry_signal" in modified_code3 - assert "force_entry_enable" in modified_code3 + # This is the 4th comment + stoploss = -0.1 + """) + # currently still missing: + # Webhook terminology, Telegram notification settings, Strategy/Config settings - assert "enter_long" in modified_code4 - assert "exit_long" in modified_code4 - assert "enter_tag" in modified_code4 + assert "populate_entry_trend" in modified_code1 + assert "populate_exit_trend" in modified_code1 + assert "check_entry_timeout" in modified_code1 + assert "check_exit_timeout" in modified_code1 + assert "custom_exit" in modified_code1 + assert "INTERFACE_VERSION = 3" in modified_code1 - assert "exit_reason" in modified_code5 + assert "timeframe" in modified_code2 + # check for not editing hyperopt spaces + assert "space='buy'" in modified_code2 + assert "space='sell'" in modified_code2 - assert "'entry': 'gtc'" in modified_code6 - assert "'exit': 'ioc'" in modified_code6 - assert "'entry': 'limit'" in modified_code6 - assert "'exit': 'market'" in modified_code6 - assert "'entry': 1" in modified_code6 - assert "'exit': 2" in modified_code6 + assert "use_exit_signal" in modified_code3 + assert "exit_profit_only" in modified_code3 + assert "exit_profit_offset" in modified_code3 + assert "ignore_roi_if_entry_signal" in modified_code3 + assert "force_entry_enable" in modified_code3 - assert "exit_reason" in modified_code7 - assert "exit_reason == 'stop_loss'" in modified_code7 + assert "enter_long" in modified_code4 + assert "exit_long" in modified_code4 + assert "enter_tag" in modified_code4 - # those tests currently don't work, next in line. - assert "exit_signal" in modified_code8 - assert "exit_reason" in modified_code8 - assert "force_exit" in modified_code8 - assert "emergency_exit" in modified_code8 + assert "exit_reason" in modified_code5 - assert "This is the 1st comment" in modified_code9 - assert "This is the 2nd comment" in modified_code9 - assert "This is the 3rd comment" in modified_code9 + assert "'entry': 'gtc'" in modified_code6 + assert "'exit': 'ioc'" in modified_code6 + assert "'entry': 'limit'" in modified_code6 + assert "'exit': 'market'" in modified_code6 + assert "'entry': 1" in modified_code6 + assert "'exit': 2" in modified_code6 + + assert "exit_reason" in modified_code7 + assert "exit_reason == 'stop_loss'" in modified_code7 + + # those tests currently don't work, next in line. + assert "exit_signal" in modified_code8 + assert "exit_reason" in modified_code8 + assert "force_exit" in modified_code8 + assert "emergency_exit" in modified_code8 + + assert "This is the 1st comment" in modified_code9 + assert "This is the 2nd comment" in modified_code9 + assert "This is the 3rd comment" in modified_code9 From a3988f56b28ddef95fd425e9804d3154491ad994 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 10 Mar 2023 09:23:56 +0100 Subject: [PATCH 22/30] Sorry matthias, did not see that you already committed something and did overwrite you. Added your version to it instead of mine and pushed again (since it was already overwritten by me). --- freqtrade/commands/strategy_utils_commands.py | 45 ++-- tests/test_strategy_updater.py | 232 +++++++++--------- 2 files changed, 138 insertions(+), 139 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 56a28cf65..ed4d0bf1a 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -20,34 +20,31 @@ def start_strategy_update(args: Dict[str, Any]) -> None: :return: None """ - if sys.version_info <= (3, 8): - print("This code requires Python 3.9 or higher. " - "We cannot continue. " - "Please upgrade your python version to use this command.") + if sys.version_info == (3, 8): # pragma: no cover + sys.exit("Freqtrade strategy updater requires Python version >= 3.9") - else: - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - strategy_objs = StrategyResolver.search_all_objects( - config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) - filtered_strategy_objs = [] - if 'strategy_list' in args: - for args_strategy in args['strategy_list']: - for strategy_obj in strategy_objs: - if (strategy_obj['name'] == args_strategy - and strategy_obj not in filtered_strategy_objs): - filtered_strategy_objs.append(strategy_obj) - break - - for filtered_strategy_obj in filtered_strategy_objs: - start_conversion(filtered_strategy_obj, config) - else: - processed_locations = set() + filtered_strategy_objs = [] + if 'strategy_list' in args: + for args_strategy in args['strategy_list']: for strategy_obj in strategy_objs: - if strategy_obj['location'] not in processed_locations: - processed_locations.add(strategy_obj['location']) - start_conversion(strategy_obj, config) + if (strategy_obj['name'] == args_strategy + and strategy_obj not in filtered_strategy_objs): + filtered_strategy_objs.append(strategy_obj) + break + + for filtered_strategy_obj in filtered_strategy_objs: + start_conversion(filtered_strategy_obj, config) + else: + processed_locations = set() + for strategy_obj in strategy_objs: + if strategy_obj['location'] not in processed_locations: + processed_locations.add(strategy_obj['location']) + start_conversion(strategy_obj, config) def start_conversion(strategy_obj, config): diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 3831c2ee6..26d173871 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -2,134 +2,136 @@ import sys +import pytest + from freqtrade.strategy.strategyupdater import StrategyUpdater def test_strategy_updater(default_conf, caplog) -> None: - if sys.version_info <= (3, 8): - print("skipped tests since python version is 3.8 or lower.") - else: - instance_strategy_updater = StrategyUpdater() - modified_code1 = instance_strategy_updater.update_code(""" - class testClass(IStrategy): - def populate_buy_trend(): - pass - def populate_sell_trend(): - pass - def check_buy_timeout(): - pass - def check_sell_timeout(): - pass - def custom_sell(): - pass - """) - modified_code2 = instance_strategy_updater.update_code(""" - ticker_interval = '15m' - buy_some_parameter = IntParameter(space='buy') - sell_some_parameter = IntParameter(space='sell') - """) - modified_code3 = instance_strategy_updater.update_code(""" - use_sell_signal = True - sell_profit_only = True - sell_profit_offset = True - ignore_roi_if_buy_signal = True - forcebuy_enable = True - """) - modified_code4 = instance_strategy_updater.update_code(""" - dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") - dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 - """) - modified_code5 = instance_strategy_updater.update_code(""" - def confirm_trade_exit(sell_reason: str): + if sys.version_info < (3, 9): + pytest.skip("StrategyUpdater is not compatible with Python 3.8", allow_module_level=True) + + instance_strategy_updater = StrategyUpdater() + modified_code1 = instance_strategy_updater.update_code(""" +class testClass(IStrategy): + def populate_buy_trend(): pass - """) - modified_code6 = instance_strategy_updater.update_code(""" - order_time_in_force = { - 'buy': 'gtc', - 'sell': 'ioc' + def populate_sell_trend(): + pass + def check_buy_timeout(): + pass + def check_sell_timeout(): + pass + def custom_sell(): + pass +""") + modified_code2 = instance_strategy_updater.update_code(""" +ticker_interval = '15m' +buy_some_parameter = IntParameter(space='buy') +sell_some_parameter = IntParameter(space='sell') +""") + modified_code3 = instance_strategy_updater.update_code(""" +use_sell_signal = True +sell_profit_only = True +sell_profit_offset = True +ignore_roi_if_buy_signal = True +forcebuy_enable = True +""") + modified_code4 = instance_strategy_updater.update_code(""" +dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") +dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 +""") + modified_code5 = instance_strategy_updater.update_code(""" +def confirm_trade_exit(sell_reason: str): + pass + """) + modified_code6 = instance_strategy_updater.update_code(""" +order_time_in_force = { + 'buy': 'gtc', + 'sell': 'ioc' +} +order_types = { + 'buy': 'limit', + 'sell': 'market', + 'stoploss': 'market', + 'stoploss_on_exchange': False +} +unfilledtimeout = { + 'buy': 1, + 'sell': 2 +} +""") + + modified_code7 = instance_strategy_updater.update_code(""" +def confirm_trade_exit(sell_reason): + if (sell_reason == 'stop_loss'): + pass +""") + modified_code8 = instance_strategy_updater.update_code(""" +sell_reason == 'sell_signal' +sell_reason == 'force_sell' +sell_reason == 'emergency_sell' +""") + modified_code9 = instance_strategy_updater.update_code(""" +# This is the 1st comment +import talib.abstract as ta +# This is the 2nd comment +import freqtrade.vendor.qtpylib.indicators as qtpylib + + +class someStrategy(IStrategy): + # This is the 3rd comment + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + "0": 0.50 } - order_types = { - 'buy': 'limit', - 'sell': 'market', - 'stoploss': 'market', - 'stoploss_on_exchange': False - } - unfilledtimeout = { - 'buy': 1, - 'sell': 2 - } - """) - modified_code7 = instance_strategy_updater.update_code(""" - def confirm_trade_exit(sell_reason): - if (sell_reason == 'stop_loss'): - pass - """) - modified_code8 = instance_strategy_updater.update_code(""" - sell_reason == 'sell_signal' - sell_reason == 'force_sell' - sell_reason == 'emergency_sell' - """) - modified_code9 = instance_strategy_updater.update_code(""" - # This is the 1st comment - import talib.abstract as ta - # This is the 2nd comment - import freqtrade.vendor.qtpylib.indicators as qtpylib + # This is the 4th comment + stoploss = -0.1 +""") + # currently still missing: + # Webhook terminology, Telegram notification settings, Strategy/Config settings + assert "populate_entry_trend" in modified_code1 + assert "populate_exit_trend" in modified_code1 + assert "check_entry_timeout" in modified_code1 + assert "check_exit_timeout" in modified_code1 + assert "custom_exit" in modified_code1 + assert "INTERFACE_VERSION = 3" in modified_code1 - class someStrategy(IStrategy): - # This is the 3rd comment - # This attribute will be overridden if the config file contains "minimal_roi" - minimal_roi = { - "0": 0.50 - } + assert "timeframe" in modified_code2 + # check for not editing hyperopt spaces + assert "space='buy'" in modified_code2 + assert "space='sell'" in modified_code2 - # This is the 4th comment - stoploss = -0.1 - """) - # currently still missing: - # Webhook terminology, Telegram notification settings, Strategy/Config settings + assert "use_exit_signal" in modified_code3 + assert "exit_profit_only" in modified_code3 + assert "exit_profit_offset" in modified_code3 + assert "ignore_roi_if_entry_signal" in modified_code3 + assert "force_entry_enable" in modified_code3 - assert "populate_entry_trend" in modified_code1 - assert "populate_exit_trend" in modified_code1 - assert "check_entry_timeout" in modified_code1 - assert "check_exit_timeout" in modified_code1 - assert "custom_exit" in modified_code1 - assert "INTERFACE_VERSION = 3" in modified_code1 + assert "enter_long" in modified_code4 + assert "exit_long" in modified_code4 + assert "enter_tag" in modified_code4 - assert "timeframe" in modified_code2 - # check for not editing hyperopt spaces - assert "space='buy'" in modified_code2 - assert "space='sell'" in modified_code2 + assert "exit_reason" in modified_code5 - assert "use_exit_signal" in modified_code3 - assert "exit_profit_only" in modified_code3 - assert "exit_profit_offset" in modified_code3 - assert "ignore_roi_if_entry_signal" in modified_code3 - assert "force_entry_enable" in modified_code3 + assert "'entry': 'gtc'" in modified_code6 + assert "'exit': 'ioc'" in modified_code6 + assert "'entry': 'limit'" in modified_code6 + assert "'exit': 'market'" in modified_code6 + assert "'entry': 1" in modified_code6 + assert "'exit': 2" in modified_code6 - assert "enter_long" in modified_code4 - assert "exit_long" in modified_code4 - assert "enter_tag" in modified_code4 + assert "exit_reason" in modified_code7 + assert "exit_reason == 'stop_loss'" in modified_code7 - assert "exit_reason" in modified_code5 + # those tests currently don't work, next in line. + assert "exit_signal" in modified_code8 + assert "exit_reason" in modified_code8 + assert "force_exit" in modified_code8 + assert "emergency_exit" in modified_code8 - assert "'entry': 'gtc'" in modified_code6 - assert "'exit': 'ioc'" in modified_code6 - assert "'entry': 'limit'" in modified_code6 - assert "'exit': 'market'" in modified_code6 - assert "'entry': 1" in modified_code6 - assert "'exit': 2" in modified_code6 - - assert "exit_reason" in modified_code7 - assert "exit_reason == 'stop_loss'" in modified_code7 - - # those tests currently don't work, next in line. - assert "exit_signal" in modified_code8 - assert "exit_reason" in modified_code8 - assert "force_exit" in modified_code8 - assert "emergency_exit" in modified_code8 - - assert "This is the 1st comment" in modified_code9 - assert "This is the 2nd comment" in modified_code9 - assert "This is the 3rd comment" in modified_code9 + assert "This is the 1st comment" in modified_code9 + assert "This is the 2nd comment" in modified_code9 + assert "This is the 3rd comment" in modified_code9 From cb086f79ff263e075dc8cb7fcd841059b7153082 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Mar 2023 15:46:34 +0100 Subject: [PATCH 23/30] Improve doc wording and command parameters --- docs/utils.md | 47 ++++++++++++++++++++++++++++----- freqtrade/commands/arguments.py | 2 +- requirements.txt | 2 +- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 3d6eda3ce..eb675442f 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -947,6 +947,7 @@ Common arguments: --userdir PATH, --user-data-dir PATH Path to userdata directory. ``` + ### Examples Print trades with id 2 and 3 as json @@ -956,11 +957,45 @@ freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print ``` ### Strategy-Updater -Updates a list strategies or all strategies within the strategies folder to be v3 compliant including futures. -If the command runs without --strategy-list then all files inside the strategies folder will be converted. -``` -usage: freqtrade strategy_updater -optional arguments: - --strategy-list defines a list of strategies that should be converted +Updates listed strategies or all strategies within the strategies folder to be v3 compliant. +If the command runs without --strategy-list then all strategies inside the strategies folder will be converted. +Your original strategy will remain available in the `user_data/strategies_orig_updater/` directory. + +!!! Warning "Conversion results" + Strategy updater will work on a "best effort" approach. Please do your due diligence and verify the results of the conversion. + We also recommend to run a python formatter (e.g. `black`) to format results in a sane manner. + +``` +usage: freqtrade strategy-updater [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] + +options: + -h, --help show this help message and exit + --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] + Provide a space-separated list of strategies to + backtest. Please note that timeframe needs to be set + either in config or via command line. When using this + together with `--export trades`, the strategy-name is + injected into the filename (so `backtest-data.json` + becomes `backtest-data-SampleStrategy.json` + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE, --log-file FILE + Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH, --data-dir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + ``` diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index a3cdc378a..9b714a864 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -116,7 +116,7 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] -ARGS_STRATEGY_UTILS = ARGS_COMMON_OPTIMIZE + ["strategy_list"] +ARGS_STRATEGY_UTILS = ["strategy_list"] class Arguments: diff --git a/requirements.txt b/requirements.txt index 7607a3664..a0ff8e03a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,4 +56,4 @@ schedule==1.1.0 websockets==10.4 janus==1.0.0 -ast-comments>=1.0.0 +ast-comments==1.0.0 From d2a412d2c65ee34576ac06638748700bb3ee4e03 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Mar 2023 15:47:03 +0100 Subject: [PATCH 24/30] Simplify start_strategy_update --- freqtrade/commands/strategy_utils_commands.py | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index ed4d0bf1a..e579ec475 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -29,31 +29,27 @@ def start_strategy_update(args: Dict[str, Any]) -> None: config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) filtered_strategy_objs = [] - if 'strategy_list' in args: - for args_strategy in args['strategy_list']: - for strategy_obj in strategy_objs: - if (strategy_obj['name'] == args_strategy - and strategy_obj not in filtered_strategy_objs): - filtered_strategy_objs.append(strategy_obj) - break + if args['strategy_list']: + filtered_strategy_objs = [ + strategy_obj for strategy_obj in strategy_objs + if strategy_obj['name'] in args['strategy_list'] + ] - for filtered_strategy_obj in filtered_strategy_objs: - start_conversion(filtered_strategy_obj, config) else: - processed_locations = set() - for strategy_obj in strategy_objs: - if strategy_obj['location'] not in processed_locations: - processed_locations.add(strategy_obj['location']) - start_conversion(strategy_obj, config) + # Use all available entries. + filtered_strategy_objs = strategy_objs + + processed_locations = set() + for strategy_obj in filtered_strategy_objs: + if strategy_obj['location'] not in processed_locations: + processed_locations.add(strategy_obj['location']) + start_conversion(strategy_obj, config) def start_conversion(strategy_obj, config): - # try: print(f"Conversion of {Path(strategy_obj['location']).name} started.") instance_strategy_updater = StrategyUpdater() start = time.perf_counter() instance_strategy_updater.start(config, strategy_obj) elapsed = time.perf_counter() - start print(f"Conversion of {Path(strategy_obj['location']).name} took {elapsed:.1f} seconds.") - # except: - # pass From 0911cd72a2fb2ed5fd87c6d3f7ad3eb2706edb6d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Mar 2023 15:59:02 +0100 Subject: [PATCH 25/30] Add test for strategy-updater start method --- freqtrade/commands/arguments.py | 2 +- tests/commands/test_commands.py | 37 ++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9b714a864..47aa37fdf 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -116,7 +116,7 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] -ARGS_STRATEGY_UTILS = ["strategy_list"] +ARGS_STRATEGY_UTILS = ["strategy_list", "strategy_path", "recursive_strategy_search"] class Arguments: diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 0ba1924a7..179712c6d 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -14,7 +14,8 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, start_new_strategy, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_strategy_update, start_test_pairlist, start_trading, + start_webserver) from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) @@ -1546,3 +1547,37 @@ def test_start_convert_db(mocker, fee, tmpdir, caplog): start_convert_db(pargs) assert db_target_file.is_file() + + +def test_start_strategy_updater(mocker, tmpdir): + sc_mock = mocker.patch('freqtrade.commands.strategy_utils_commands.start_conversion') + teststrats = Path(__file__).parent.parent / 'strategy/strats' + args = [ + "strategy-updater", + "--userdir", + str(tmpdir), + "--strategy-path", + str(teststrats), + ] + pargs = get_args(args) + pargs['config'] = None + start_strategy_update(pargs) + # Number of strategies in the test directory + assert sc_mock.call_count == 11 + + sc_mock.reset_mock() + args = [ + "strategy-updater", + "--userdir", + str(tmpdir), + "--strategy-path", + str(teststrats), + "--strategy-list", + "StrategyTestV3", + "StrategyTestV2" + ] + pargs = get_args(args) + pargs['config'] = None + start_strategy_update(pargs) + # Number of strategies in the test directory + assert sc_mock.call_count == 2 From b5c4f9ebe293097c08554e06c785a3e40ed993b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Mar 2023 16:26:12 +0100 Subject: [PATCH 26/30] Split updater_tests to be clearer --- tests/test_strategy_updater.py | 139 ++++++++++++++++++++------------- 1 file changed, 86 insertions(+), 53 deletions(-) diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 26d173871..7205a609d 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -7,9 +7,11 @@ import pytest from freqtrade.strategy.strategyupdater import StrategyUpdater -def test_strategy_updater(default_conf, caplog) -> None: - if sys.version_info < (3, 9): - pytest.skip("StrategyUpdater is not compatible with Python 3.8", allow_module_level=True) +if sys.version_info < (3, 9): + pytest.skip("StrategyUpdater is not compatible with Python 3.8", allow_module_level=True) + + +def test_strategy_updater_methods(default_conf, caplog) -> None: instance_strategy_updater = StrategyUpdater() modified_code1 = instance_strategy_updater.update_code(""" @@ -25,11 +27,32 @@ class testClass(IStrategy): def custom_sell(): pass """) + + assert "populate_entry_trend" in modified_code1 + assert "populate_exit_trend" in modified_code1 + assert "check_entry_timeout" in modified_code1 + assert "check_exit_timeout" in modified_code1 + assert "custom_exit" in modified_code1 + assert "INTERFACE_VERSION = 3" in modified_code1 + + +def test_strategy_updater_params(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code2 = instance_strategy_updater.update_code(""" ticker_interval = '15m' buy_some_parameter = IntParameter(space='buy') sell_some_parameter = IntParameter(space='sell') """) + + assert "timeframe" in modified_code2 + # check for not editing hyperopt spaces + assert "space='buy'" in modified_code2 + assert "space='sell'" in modified_code2 + + +def test_strategy_updater_constants(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() modified_code3 = instance_strategy_updater.update_code(""" use_sell_signal = True sell_profit_only = True @@ -37,15 +60,38 @@ sell_profit_offset = True ignore_roi_if_buy_signal = True forcebuy_enable = True """) - modified_code4 = instance_strategy_updater.update_code(""" + + assert "use_exit_signal" in modified_code3 + assert "exit_profit_only" in modified_code3 + assert "exit_profit_offset" in modified_code3 + assert "ignore_roi_if_entry_signal" in modified_code3 + assert "force_entry_enable" in modified_code3 + + +def test_strategy_updater_df_columns(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 """) - modified_code5 = instance_strategy_updater.update_code(""" + + assert "enter_long" in modified_code + assert "exit_long" in modified_code + assert "enter_tag" in modified_code + + +def test_strategy_updater_method_params(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" def confirm_trade_exit(sell_reason: str): pass """) - modified_code6 = instance_strategy_updater.update_code(""" + assert "exit_reason" in modified_code + + +def test_strategy_updater_dicts(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" order_time_in_force = { 'buy': 'gtc', 'sell': 'ioc' @@ -62,17 +108,44 @@ unfilledtimeout = { } """) - modified_code7 = instance_strategy_updater.update_code(""" + assert "'entry': 'gtc'" in modified_code + assert "'exit': 'ioc'" in modified_code + assert "'entry': 'limit'" in modified_code + assert "'exit': 'market'" in modified_code + assert "'entry': 1" in modified_code + assert "'exit': 2" in modified_code + + +def test_strategy_updater_comparisons(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" def confirm_trade_exit(sell_reason): if (sell_reason == 'stop_loss'): pass """) - modified_code8 = instance_strategy_updater.update_code(""" + assert "exit_reason" in modified_code + assert "exit_reason == 'stop_loss'" in modified_code + + +def test_strategy_updater_strings(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + + modified_code = instance_strategy_updater.update_code(""" sell_reason == 'sell_signal' sell_reason == 'force_sell' sell_reason == 'emergency_sell' """) - modified_code9 = instance_strategy_updater.update_code(""" + + # those tests currently don't work, next in line. + assert "exit_signal" in modified_code + assert "exit_reason" in modified_code + assert "force_exit" in modified_code + assert "emergency_exit" in modified_code + + +def test_strategy_updater_comments(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" # This is the 1st comment import talib.abstract as ta # This is the 2nd comment @@ -89,49 +162,9 @@ class someStrategy(IStrategy): # This is the 4th comment stoploss = -0.1 """) + + assert "This is the 1st comment" in modified_code + assert "This is the 2nd comment" in modified_code + assert "This is the 3rd comment" in modified_code # currently still missing: # Webhook terminology, Telegram notification settings, Strategy/Config settings - - assert "populate_entry_trend" in modified_code1 - assert "populate_exit_trend" in modified_code1 - assert "check_entry_timeout" in modified_code1 - assert "check_exit_timeout" in modified_code1 - assert "custom_exit" in modified_code1 - assert "INTERFACE_VERSION = 3" in modified_code1 - - assert "timeframe" in modified_code2 - # check for not editing hyperopt spaces - assert "space='buy'" in modified_code2 - assert "space='sell'" in modified_code2 - - assert "use_exit_signal" in modified_code3 - assert "exit_profit_only" in modified_code3 - assert "exit_profit_offset" in modified_code3 - assert "ignore_roi_if_entry_signal" in modified_code3 - assert "force_entry_enable" in modified_code3 - - assert "enter_long" in modified_code4 - assert "exit_long" in modified_code4 - assert "enter_tag" in modified_code4 - - assert "exit_reason" in modified_code5 - - assert "'entry': 'gtc'" in modified_code6 - assert "'exit': 'ioc'" in modified_code6 - assert "'entry': 'limit'" in modified_code6 - assert "'exit': 'market'" in modified_code6 - assert "'entry': 1" in modified_code6 - assert "'exit': 2" in modified_code6 - - assert "exit_reason" in modified_code7 - assert "exit_reason == 'stop_loss'" in modified_code7 - - # those tests currently don't work, next in line. - assert "exit_signal" in modified_code8 - assert "exit_reason" in modified_code8 - assert "force_exit" in modified_code8 - assert "emergency_exit" in modified_code8 - - assert "This is the 1st comment" in modified_code9 - assert "This is the 2nd comment" in modified_code9 - assert "This is the 3rd comment" in modified_code9 From f5848ea891723fcdf33e8794f2018c0547f8252d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Mar 2023 16:29:18 +0100 Subject: [PATCH 27/30] Add test for successful_buys --- tests/test_strategy_updater.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 7205a609d..4aad152bf 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -84,9 +84,11 @@ def test_strategy_updater_method_params(default_conf, caplog) -> None: instance_strategy_updater = StrategyUpdater() modified_code = instance_strategy_updater.update_code(""" def confirm_trade_exit(sell_reason: str): + nr_orders = trades.nr_of_successful_buys pass """) assert "exit_reason" in modified_code + assert "nr_orders = trades.nr_of_successful_entries" in modified_code def test_strategy_updater_dicts(default_conf, caplog) -> None: From f584edf809c5413f7136ef7c7e1cda5888f5363d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Mar 2023 16:45:56 +0100 Subject: [PATCH 28/30] Improve tests by simply running a full strategy through everything --- freqtrade/strategy/strategyupdater.py | 4 ++- tests/commands/test_commands.py | 2 +- tests/test_strategy_updater.py | 42 +++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index 6fe1f326c..bc692b71c 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -3,6 +3,8 @@ from pathlib import Path import ast_comments +from freqtrade.constants import Config + class StrategyUpdater: name_mapping = { @@ -42,7 +44,7 @@ class StrategyUpdater: # create a dictionary that maps the old column names to the new ones rename_dict = {'buy': 'enter_long', 'sell': 'exit_long', 'buy_tag': 'enter_tag'} - def start(self, config, strategy_obj: dict) -> None: + def start(self, config: Config, strategy_obj: dict) -> None: """ Run strategy updater It updates a strategy to v3 with the help of the ast-module diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 179712c6d..318590b32 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1549,7 +1549,7 @@ def test_start_convert_db(mocker, fee, tmpdir, caplog): assert db_target_file.is_file() -def test_start_strategy_updater(mocker, tmpdir): +def test_start_strategy_updater(mocker, tmpdir): sc_mock = mocker.patch('freqtrade.commands.strategy_utils_commands.start_conversion') teststrats = Path(__file__).parent.parent / 'strategy/strats' args = [ diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index 4aad152bf..d3bdd27b5 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -1,16 +1,56 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name +import re +import shutil import sys +from pathlib import Path import pytest +from freqtrade.commands.strategy_utils_commands import start_strategy_update from freqtrade.strategy.strategyupdater import StrategyUpdater +from tests.conftest import get_args if sys.version_info < (3, 9): pytest.skip("StrategyUpdater is not compatible with Python 3.8", allow_module_level=True) +def test_strategy_updater_start(tmpdir, capsys) -> None: + # Effective test without mocks. + teststrats = Path(__file__).parent / 'strategy/strats' + tmpdirp = Path(tmpdir) / "strategies" + tmpdirp.mkdir() + shutil.copy(teststrats / 'strategy_test_v2.py', tmpdirp) + old_code = (teststrats / 'strategy_test_v2.py').read_text() + + args = [ + "strategy-updater", + "--userdir", + str(tmpdir), + "--strategy-list", + "StrategyTestV2" + ] + pargs = get_args(args) + pargs['config'] = None + + start_strategy_update(pargs) + + assert Path(tmpdir / "strategies_orig_updater").exists() + # Backup file exists + assert Path(tmpdir / "strategies_orig_updater" / 'strategy_test_v2.py').exists() + # updated file exists + new_file = Path(tmpdirp / 'strategy_test_v2.py') + assert new_file.exists() + new_code = new_file.read_text() + assert 'INTERFACE_VERSION = 3' in new_code + assert 'INTERFACE_VERSION = 2' in old_code + captured = capsys.readouterr() + + assert 'Conversion of strategy_test_v2.py started.' in captured.out + assert re.search(r'Conversion of strategy_test_v2\.py took .* seconds', captured.out) + + def test_strategy_updater_methods(default_conf, caplog) -> None: instance_strategy_updater = StrategyUpdater() @@ -155,6 +195,7 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib class someStrategy(IStrategy): + INTERFACE_VERSION = 2 # This is the 3rd comment # This attribute will be overridden if the config file contains "minimal_roi" minimal_roi = { @@ -168,5 +209,6 @@ class someStrategy(IStrategy): assert "This is the 1st comment" in modified_code assert "This is the 2nd comment" in modified_code assert "This is the 3rd comment" in modified_code + assert "INTERFACE_VERSION = 3" in modified_code # currently still missing: # Webhook terminology, Telegram notification settings, Strategy/Config settings From b1f88e88612285c5ccaa219c6d9366a575c30de9 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 18 Mar 2023 20:02:55 +0100 Subject: [PATCH 29/30] fixed typo from trades to trade --- freqtrade/strategy/strategyupdater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py index bc692b71c..2669dcc4a 100644 --- a/freqtrade/strategy/strategyupdater.py +++ b/freqtrade/strategy/strategyupdater.py @@ -181,7 +181,7 @@ class NameUpdater(ast_comments.NodeTransformer): def visit_Attribute(self, node): if ( isinstance(node.value, ast_comments.Name) - and node.value.id == 'trades' + and node.value.id == 'trade' and node.attr == 'nr_of_successful_buys' ): node.attr = 'nr_of_successful_entries' From bf3f2e4de4cdaf52f41e55ef65ab997a5c29e50d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Mar 2023 11:16:54 +0100 Subject: [PATCH 30/30] Fix failing test --- tests/test_strategy_updater.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py index d3bdd27b5..597d49fda 100644 --- a/tests/test_strategy_updater.py +++ b/tests/test_strategy_updater.py @@ -124,11 +124,11 @@ def test_strategy_updater_method_params(default_conf, caplog) -> None: instance_strategy_updater = StrategyUpdater() modified_code = instance_strategy_updater.update_code(""" def confirm_trade_exit(sell_reason: str): - nr_orders = trades.nr_of_successful_buys + nr_orders = trade.nr_of_successful_buys pass """) assert "exit_reason" in modified_code - assert "nr_orders = trades.nr_of_successful_entries" in modified_code + assert "nr_orders = trade.nr_of_successful_entries" in modified_code def test_strategy_updater_dicts(default_conf, caplog) -> None: