diff --git a/docs/utils.md b/docs/utils.md index 87c7f6aa6..eb675442f 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -955,3 +955,47 @@ Print trades with id 2 and 3 as json ``` bash freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json ``` + +### Strategy-Updater + +Updates listed strategies or all strategies within the strategies folder to be v3 compliant. +If the command runs without --strategy-list then all strategies inside the strategies folder will be converted. +Your original strategy will remain available in the `user_data/strategies_orig_updater/` directory. + +!!! Warning "Conversion results" + Strategy updater will work on a "best effort" approach. Please do your due diligence and verify the results of the conversion. + We also recommend to run a python formatter (e.g. `black`) to format results in a sane manner. + +``` +usage: freqtrade strategy-updater [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] + +options: + -h, --help show this help message and exit + --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] + Provide a space-separated list of strategies to + backtest. Please note that timeframe needs to be set + either in config or via command line. When using this + together with `--export trades`, the strategy-name is + injected into the filename (so `backtest-data.json` + becomes `backtest-data-SampleStrategy.json` + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE, --log-file FILE + Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH, --data-dir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 788657cc8..66a9c995b 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -22,5 +22,6 @@ from freqtrade.commands.optimize_commands import (start_backtesting, start_backt start_edge, start_hyperopt) from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit +from freqtrade.commands.strategy_utils_commands import start_strategy_update from freqtrade.commands.trade_commands import start_trading from freqtrade.commands.webserver_commands import start_webserver diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index b53a1022d..47aa37fdf 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -111,10 +111,13 @@ ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-freqaimodels", "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter", - "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"] + "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv", + "strategy-updater"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] +ARGS_STRATEGY_UTILS = ["strategy_list", "strategy_path", "recursive_strategy_search"] + class Arguments: """ @@ -198,8 +201,8 @@ class Arguments: start_list_freqAI_models, start_list_markets, start_list_strategies, start_list_timeframes, start_new_config, start_new_strategy, start_plot_dataframe, - start_plot_profit, start_show_trades, start_test_pairlist, - start_trading, start_webserver) + start_plot_profit, start_show_trades, start_strategy_update, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -440,3 +443,11 @@ class Arguments: parents=[_common_parser]) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) + + # Add strategy_updater subcommand + strategy_updater_cmd = subparsers.add_parser('strategy-updater', + help='updates outdated strategy' + 'files to the current version', + parents=[_common_parser]) + strategy_updater_cmd.set_defaults(func=start_strategy_update) + self._build_args(optionlist=ARGS_STRATEGY_UTILS, parser=strategy_updater_cmd) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py new file mode 100644 index 000000000..e579ec475 --- /dev/null +++ b/freqtrade/commands/strategy_utils_commands.py @@ -0,0 +1,55 @@ +import logging +import sys +import time +from pathlib import Path +from typing import Any, Dict + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode +from freqtrade.resolvers import StrategyResolver +from freqtrade.strategy.strategyupdater import StrategyUpdater + + +logger = logging.getLogger(__name__) + + +def start_strategy_update(args: Dict[str, Any]) -> None: + """ + Start the strategy updating script + :param args: Cli args from Arguments() + :return: None + """ + + if sys.version_info == (3, 8): # pragma: no cover + sys.exit("Freqtrade strategy updater requires Python version >= 3.9") + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + + filtered_strategy_objs = [] + if args['strategy_list']: + filtered_strategy_objs = [ + strategy_obj for strategy_obj in strategy_objs + if strategy_obj['name'] in args['strategy_list'] + ] + + else: + # Use all available entries. + filtered_strategy_objs = strategy_objs + + processed_locations = set() + for strategy_obj in filtered_strategy_objs: + if strategy_obj['location'] not in processed_locations: + processed_locations.add(strategy_obj['location']) + start_conversion(strategy_obj, config) + + +def start_conversion(strategy_obj, config): + print(f"Conversion of {Path(strategy_obj['location']).name} started.") + instance_strategy_updater = StrategyUpdater() + start = time.perf_counter() + instance_strategy_updater.start(config, strategy_obj) + elapsed = time.perf_counter() - start + print(f"Conversion of {Path(strategy_obj['location']).name} took {elapsed:.1f} seconds.") diff --git a/freqtrade/strategy/strategyupdater.py b/freqtrade/strategy/strategyupdater.py new file mode 100644 index 000000000..2669dcc4a --- /dev/null +++ b/freqtrade/strategy/strategyupdater.py @@ -0,0 +1,255 @@ +import shutil +from pathlib import Path + +import ast_comments + +from freqtrade.constants import Config + + +class StrategyUpdater: + name_mapping = { + 'ticker_interval': 'timeframe', + 'buy': 'enter_long', + 'sell': 'exit_long', + 'buy_tag': 'enter_tag', + 'sell_reason': 'exit_reason', + + 'sell_signal': 'exit_signal', + 'custom_sell': 'custom_exit', + 'force_sell': 'force_exit', + 'emergency_sell': 'emergency_exit', + + # Strategy/config settings: + 'use_sell_signal': 'use_exit_signal', + 'sell_profit_only': 'exit_profit_only', + 'sell_profit_offset': 'exit_profit_offset', + 'ignore_roi_if_buy_signal': 'ignore_roi_if_entry_signal', + 'forcebuy_enable': 'force_entry_enable', + } + + function_mapping = { + 'populate_buy_trend': 'populate_entry_trend', + 'populate_sell_trend': 'populate_exit_trend', + 'custom_sell': 'custom_exit', + 'check_buy_timeout': 'check_entry_timeout', + 'check_sell_timeout': 'check_exit_timeout', + # '': '', + } + # order_time_in_force, order_types, unfilledtimeout + otif_ot_unfilledtimeout = { + 'buy': 'entry', + 'sell': 'exit', + } + + # create a dictionary that maps the old column names to the new ones + rename_dict = {'buy': 'enter_long', 'sell': 'exit_long', 'buy_tag': 'enter_tag'} + + def start(self, config: Config, strategy_obj: dict) -> None: + """ + Run strategy updater + It updates a strategy to v3 with the help of the ast-module + :return: None + """ + + source_file = strategy_obj['location'] + strategies_backup_folder = Path.joinpath(config['user_data_dir'], "strategies_orig_updater") + target_file = Path.joinpath(strategies_backup_folder, strategy_obj['location_rel']) + + # read the file + with Path(source_file).open('r') as f: + old_code = f.read() + if not strategies_backup_folder.is_dir(): + Path(strategies_backup_folder).mkdir(parents=True, exist_ok=True) + + # backup original + # => currently no date after the filename, + # could get overridden pretty fast if this is fired twice! + # The folder is always the same and the file name too (currently). + shutil.copy(source_file, target_file) + + # update the code + new_code = self.update_code(old_code) + # write the modified code to the destination folder + with Path(source_file).open('w') as f: + f.write(new_code) + + # define the function to update the code + def update_code(self, code): + # parse the code into an AST + tree = ast_comments.parse(code) + + # use the AST to update the code + updated_code = self.modify_ast(tree) + + # return the modified code without executing it + return updated_code + + # function that uses the ast module to update the code + def modify_ast(self, tree): # noqa + # use the visitor to update the names and functions in the AST + NameUpdater().visit(tree) + + # first fix the comments, so it understands "\n" properly inside multi line comments. + ast_comments.fix_missing_locations(tree) + ast_comments.increment_lineno(tree, n=1) + + # generate the new code from the updated AST + # without indent {} parameters would just be written straight one after the other. + + # ast_comments would be amazing since this is the only solution that carries over comments, + # but it does currently not have an unparse function, hopefully in the future ... ! + # return ast_comments.unparse(tree) + + return ast_comments.unparse(tree) + + +# Here we go through each respective node, slice, elt, key ... to replace outdated entries. +class NameUpdater(ast_comments.NodeTransformer): + def generic_visit(self, node): + + # space is not yet transferred from buy/sell to entry/exit and thereby has to be skipped. + if isinstance(node, ast_comments.keyword): + if node.arg == "space": + return node + + # from here on this is the original function. + for field, old_value in ast_comments.iter_fields(node): + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, ast_comments.AST): + value = self.visit(value) + if value is None: + continue + elif not isinstance(value, ast_comments.AST): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, ast_comments.AST): + new_node = self.visit(old_value) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node + + def visit_Expr(self, node): + if hasattr(node.value, "left") and hasattr(node.value.left, "id"): + node.value.left.id = self.check_dict(StrategyUpdater.name_mapping, node.value.left.id) + self.visit(node.value) + return node + + # Renames an element if contained inside a dictionary. + @staticmethod + def check_dict(current_dict: dict, element: str): + if element in current_dict: + element = current_dict[element] + return element + + def visit_arguments(self, node): + if isinstance(node.args, list): + for arg in node.args: + arg.arg = self.check_dict(StrategyUpdater.name_mapping, arg.arg) + return node + + def visit_Name(self, node): + # if the name is in the mapping, update it + node.id = self.check_dict(StrategyUpdater.name_mapping, node.id) + return node + + def visit_Import(self, node): + # do not update the names in import statements + return node + + def visit_ImportFrom(self, node): + # if hasattr(node, "module"): + # if node.module == "freqtrade.strategy.hyper": + # node.module = "freqtrade.strategy" + return node + + def visit_If(self, node: ast_comments.If): + for child in ast_comments.iter_child_nodes(node): + self.visit(child) + return node + + def visit_FunctionDef(self, node): + node.name = self.check_dict(StrategyUpdater.function_mapping, node.name) + self.generic_visit(node) + return node + + def visit_Attribute(self, node): + if ( + isinstance(node.value, ast_comments.Name) + and node.value.id == 'trade' + and node.attr == 'nr_of_successful_buys' + ): + node.attr = 'nr_of_successful_entries' + return node + + def visit_ClassDef(self, node): + # check if the class is derived from IStrategy + if any(isinstance(base, ast_comments.Name) and + base.id == 'IStrategy' for base in node.bases): + # check if the INTERFACE_VERSION variable exists + has_interface_version = any( + isinstance(child, ast_comments.Assign) and + isinstance(child.targets[0], ast_comments.Name) and + child.targets[0].id == 'INTERFACE_VERSION' + for child in node.body + ) + + # if the INTERFACE_VERSION variable does not exist, add it as the first child + if not has_interface_version: + node.body.insert(0, ast_comments.parse('INTERFACE_VERSION = 3').body[0]) + # otherwise, update its value to 3 + else: + for child in node.body: + if ( + isinstance(child, ast_comments.Assign) + and isinstance(child.targets[0], ast_comments.Name) + and child.targets[0].id == 'INTERFACE_VERSION' + ): + child.value = ast_comments.parse('3').body[0].value + self.generic_visit(node) + return node + + def visit_Subscript(self, node): + if isinstance(node.slice, ast_comments.Constant): + if node.slice.value in StrategyUpdater.rename_dict: + # Replace the slice attributes with the values from rename_dict + node.slice.value = StrategyUpdater.rename_dict[node.slice.value] + if hasattr(node.slice, "elts"): + self.visit_elts(node.slice.elts) + if hasattr(node.slice, "value"): + if hasattr(node.slice.value, "elts"): + self.visit_elts(node.slice.value.elts) + return node + + # elts can have elts (technically recursively) + def visit_elts(self, elts): + if isinstance(elts, list): + for elt in elts: + self.visit_elt(elt) + else: + self.visit_elt(elts) + return elts + + # sub function again needed since the structure itself is highly flexible ... + def visit_elt(self, elt): + if isinstance(elt, ast_comments.Constant) and elt.value in StrategyUpdater.rename_dict: + elt.value = StrategyUpdater.rename_dict[elt.value] + if hasattr(elt, "elts"): + self.visit_elts(elt.elts) + if hasattr(elt, "args"): + if isinstance(elt.args, ast_comments.arguments): + self.visit_elts(elt.args) + else: + for arg in elt.args: + self.visit_elts(arg) + return elt + + def visit_Constant(self, node): + node.value = self.check_dict(StrategyUpdater.otif_ot_unfilledtimeout, node.value) + node.value = self.check_dict(StrategyUpdater.name_mapping, node.value) + return node diff --git a/requirements.txt b/requirements.txt index 868fc9699..9e17424f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,3 +55,5 @@ schedule==1.1.0 #WS Messages websockets==10.4 janus==1.0.0 + +ast-comments==1.0.0 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 0ba1924a7..318590b32 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -14,7 +14,8 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, start_new_strategy, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_strategy_update, start_test_pairlist, start_trading, + start_webserver) from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) @@ -1546,3 +1547,37 @@ def test_start_convert_db(mocker, fee, tmpdir, caplog): start_convert_db(pargs) assert db_target_file.is_file() + + +def test_start_strategy_updater(mocker, tmpdir): + sc_mock = mocker.patch('freqtrade.commands.strategy_utils_commands.start_conversion') + teststrats = Path(__file__).parent.parent / 'strategy/strats' + args = [ + "strategy-updater", + "--userdir", + str(tmpdir), + "--strategy-path", + str(teststrats), + ] + pargs = get_args(args) + pargs['config'] = None + start_strategy_update(pargs) + # Number of strategies in the test directory + assert sc_mock.call_count == 11 + + sc_mock.reset_mock() + args = [ + "strategy-updater", + "--userdir", + str(tmpdir), + "--strategy-path", + str(teststrats), + "--strategy-list", + "StrategyTestV3", + "StrategyTestV2" + ] + pargs = get_args(args) + pargs['config'] = None + start_strategy_update(pargs) + # Number of strategies in the test directory + assert sc_mock.call_count == 2 diff --git a/tests/test_strategy_updater.py b/tests/test_strategy_updater.py new file mode 100644 index 000000000..597d49fda --- /dev/null +++ b/tests/test_strategy_updater.py @@ -0,0 +1,214 @@ +# pragma pylint: disable=missing-docstring, protected-access, invalid-name + +import re +import shutil +import sys +from pathlib import Path + +import pytest + +from freqtrade.commands.strategy_utils_commands import start_strategy_update +from freqtrade.strategy.strategyupdater import StrategyUpdater +from tests.conftest import get_args + + +if sys.version_info < (3, 9): + pytest.skip("StrategyUpdater is not compatible with Python 3.8", allow_module_level=True) + + +def test_strategy_updater_start(tmpdir, capsys) -> None: + # Effective test without mocks. + teststrats = Path(__file__).parent / 'strategy/strats' + tmpdirp = Path(tmpdir) / "strategies" + tmpdirp.mkdir() + shutil.copy(teststrats / 'strategy_test_v2.py', tmpdirp) + old_code = (teststrats / 'strategy_test_v2.py').read_text() + + args = [ + "strategy-updater", + "--userdir", + str(tmpdir), + "--strategy-list", + "StrategyTestV2" + ] + pargs = get_args(args) + pargs['config'] = None + + start_strategy_update(pargs) + + assert Path(tmpdir / "strategies_orig_updater").exists() + # Backup file exists + assert Path(tmpdir / "strategies_orig_updater" / 'strategy_test_v2.py').exists() + # updated file exists + new_file = Path(tmpdirp / 'strategy_test_v2.py') + assert new_file.exists() + new_code = new_file.read_text() + assert 'INTERFACE_VERSION = 3' in new_code + assert 'INTERFACE_VERSION = 2' in old_code + captured = capsys.readouterr() + + assert 'Conversion of strategy_test_v2.py started.' in captured.out + assert re.search(r'Conversion of strategy_test_v2\.py took .* seconds', captured.out) + + +def test_strategy_updater_methods(default_conf, caplog) -> None: + + instance_strategy_updater = StrategyUpdater() + modified_code1 = instance_strategy_updater.update_code(""" +class testClass(IStrategy): + def populate_buy_trend(): + pass + def populate_sell_trend(): + pass + def check_buy_timeout(): + pass + def check_sell_timeout(): + pass + def custom_sell(): + pass +""") + + assert "populate_entry_trend" in modified_code1 + assert "populate_exit_trend" in modified_code1 + assert "check_entry_timeout" in modified_code1 + assert "check_exit_timeout" in modified_code1 + assert "custom_exit" in modified_code1 + assert "INTERFACE_VERSION = 3" in modified_code1 + + +def test_strategy_updater_params(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + + modified_code2 = instance_strategy_updater.update_code(""" +ticker_interval = '15m' +buy_some_parameter = IntParameter(space='buy') +sell_some_parameter = IntParameter(space='sell') +""") + + assert "timeframe" in modified_code2 + # check for not editing hyperopt spaces + assert "space='buy'" in modified_code2 + assert "space='sell'" in modified_code2 + + +def test_strategy_updater_constants(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code3 = instance_strategy_updater.update_code(""" +use_sell_signal = True +sell_profit_only = True +sell_profit_offset = True +ignore_roi_if_buy_signal = True +forcebuy_enable = True +""") + + assert "use_exit_signal" in modified_code3 + assert "exit_profit_only" in modified_code3 + assert "exit_profit_offset" in modified_code3 + assert "ignore_roi_if_entry_signal" in modified_code3 + assert "force_entry_enable" in modified_code3 + + +def test_strategy_updater_df_columns(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" +dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1") +dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1 +""") + + assert "enter_long" in modified_code + assert "exit_long" in modified_code + assert "enter_tag" in modified_code + + +def test_strategy_updater_method_params(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" +def confirm_trade_exit(sell_reason: str): + nr_orders = trade.nr_of_successful_buys + pass + """) + assert "exit_reason" in modified_code + assert "nr_orders = trade.nr_of_successful_entries" in modified_code + + +def test_strategy_updater_dicts(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" +order_time_in_force = { + 'buy': 'gtc', + 'sell': 'ioc' +} +order_types = { + 'buy': 'limit', + 'sell': 'market', + 'stoploss': 'market', + 'stoploss_on_exchange': False +} +unfilledtimeout = { + 'buy': 1, + 'sell': 2 +} +""") + + assert "'entry': 'gtc'" in modified_code + assert "'exit': 'ioc'" in modified_code + assert "'entry': 'limit'" in modified_code + assert "'exit': 'market'" in modified_code + assert "'entry': 1" in modified_code + assert "'exit': 2" in modified_code + + +def test_strategy_updater_comparisons(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" +def confirm_trade_exit(sell_reason): + if (sell_reason == 'stop_loss'): + pass +""") + assert "exit_reason" in modified_code + assert "exit_reason == 'stop_loss'" in modified_code + + +def test_strategy_updater_strings(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + + modified_code = instance_strategy_updater.update_code(""" +sell_reason == 'sell_signal' +sell_reason == 'force_sell' +sell_reason == 'emergency_sell' +""") + + # those tests currently don't work, next in line. + assert "exit_signal" in modified_code + assert "exit_reason" in modified_code + assert "force_exit" in modified_code + assert "emergency_exit" in modified_code + + +def test_strategy_updater_comments(default_conf, caplog) -> None: + instance_strategy_updater = StrategyUpdater() + modified_code = instance_strategy_updater.update_code(""" +# This is the 1st comment +import talib.abstract as ta +# This is the 2nd comment +import freqtrade.vendor.qtpylib.indicators as qtpylib + + +class someStrategy(IStrategy): + INTERFACE_VERSION = 2 + # This is the 3rd comment + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + "0": 0.50 + } + + # This is the 4th comment + stoploss = -0.1 +""") + + assert "This is the 1st comment" in modified_code + assert "This is the 2nd comment" in modified_code + assert "This is the 3rd comment" in modified_code + assert "INTERFACE_VERSION = 3" in modified_code + # currently still missing: + # Webhook terminology, Telegram notification settings, Strategy/Config settings diff --git a/user_data/strategies/.gitkeep b/user_data/strategies/.gitkeep deleted file mode 100644 index e69de29bb..000000000