Merge pull request #7951 from hippocritical/strategy_utils

strategy_updater
This commit is contained in:
Matthias 2023-03-19 14:28:36 +01:00 committed by GitHub
commit 7aa56adf15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 621 additions and 4 deletions

View File

@ -955,3 +955,47 @@ Print trades with id 2 and 3 as json
``` bash ``` bash
freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json
``` ```
### Strategy-Updater
Updates listed strategies or all strategies within the strategies folder to be v3 compliant.
If the command runs without --strategy-list then all strategies inside the strategies folder will be converted.
Your original strategy will remain available in the `user_data/strategies_orig_updater/` directory.
!!! Warning "Conversion results"
Strategy updater will work on a "best effort" approach. Please do your due diligence and verify the results of the conversion.
We also recommend to run a python formatter (e.g. `black`) to format results in a sane manner.
```
usage: freqtrade strategy-updater [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-d PATH] [--userdir PATH]
[--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
options:
-h, --help show this help message and exit
--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
Provide a space-separated list of strategies to
backtest. Please note that timeframe needs to be set
either in config or via command line. When using this
together with `--export trades`, the strategy-name is
injected into the filename (so `backtest-data.json`
becomes `backtest-data-SampleStrategy.json`
Common arguments:
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).
--logfile FILE, --log-file FILE
Log to the file specified. Special values are:
'syslog', 'journald'. See the documentation for more
details.
-V, --version show program's version number and exit
-c PATH, --config PATH
Specify configuration file (default:
`userdir/config.json` or `config.json` whichever
exists). Multiple --config options may be used. Can be
set to `-` to read config from stdin.
-d PATH, --datadir PATH, --data-dir PATH
Path to directory with historical backtesting data.
--userdir PATH, --user-data-dir PATH
Path to userdata directory.
```

View File

@ -22,5 +22,6 @@ from freqtrade.commands.optimize_commands import (start_backtesting, start_backt
start_edge, start_hyperopt) start_edge, start_hyperopt)
from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.pairlist_commands import start_test_pairlist
from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
from freqtrade.commands.strategy_utils_commands import start_strategy_update
from freqtrade.commands.trade_commands import start_trading from freqtrade.commands.trade_commands import start_trading
from freqtrade.commands.webserver_commands import start_webserver from freqtrade.commands.webserver_commands import start_webserver

View File

@ -111,10 +111,13 @@ ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
"list-markets", "list-pairs", "list-strategies", "list-freqaimodels", "list-markets", "list-pairs", "list-strategies", "list-freqaimodels",
"list-data", "hyperopt-list", "hyperopt-show", "backtest-filter", "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter",
"plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"] "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv",
"strategy-updater"]
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
ARGS_STRATEGY_UTILS = ["strategy_list", "strategy_path", "recursive_strategy_search"]
class Arguments: class Arguments:
""" """
@ -198,8 +201,8 @@ class Arguments:
start_list_freqAI_models, start_list_markets, start_list_freqAI_models, start_list_markets,
start_list_strategies, start_list_timeframes, start_list_strategies, start_list_timeframes,
start_new_config, start_new_strategy, start_plot_dataframe, start_new_config, start_new_strategy, start_plot_dataframe,
start_plot_profit, start_show_trades, start_test_pairlist, start_plot_profit, start_show_trades, start_strategy_update,
start_trading, start_webserver) start_test_pairlist, start_trading, start_webserver)
subparsers = self.parser.add_subparsers(dest='command', subparsers = self.parser.add_subparsers(dest='command',
# Use custom message when no subhandler is added # Use custom message when no subhandler is added
@ -440,3 +443,11 @@ class Arguments:
parents=[_common_parser]) parents=[_common_parser])
webserver_cmd.set_defaults(func=start_webserver) webserver_cmd.set_defaults(func=start_webserver)
self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)
# Add strategy_updater subcommand
strategy_updater_cmd = subparsers.add_parser('strategy-updater',
help='updates outdated strategy'
'files to the current version',
parents=[_common_parser])
strategy_updater_cmd.set_defaults(func=start_strategy_update)
self._build_args(optionlist=ARGS_STRATEGY_UTILS, parser=strategy_updater_cmd)

View File

@ -0,0 +1,55 @@
import logging
import sys
import time
from pathlib import Path
from typing import Any, Dict
from freqtrade.configuration import setup_utils_configuration
from freqtrade.enums import RunMode
from freqtrade.resolvers import StrategyResolver
from freqtrade.strategy.strategyupdater import StrategyUpdater
logger = logging.getLogger(__name__)
def start_strategy_update(args: Dict[str, Any]) -> None:
"""
Start the strategy updating script
:param args: Cli args from Arguments()
:return: None
"""
if sys.version_info == (3, 8): # pragma: no cover
sys.exit("Freqtrade strategy updater requires Python version >= 3.9")
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
strategy_objs = StrategyResolver.search_all_objects(
config, enum_failed=False, recursive=config.get('recursive_strategy_search', False))
filtered_strategy_objs = []
if args['strategy_list']:
filtered_strategy_objs = [
strategy_obj for strategy_obj in strategy_objs
if strategy_obj['name'] in args['strategy_list']
]
else:
# Use all available entries.
filtered_strategy_objs = strategy_objs
processed_locations = set()
for strategy_obj in filtered_strategy_objs:
if strategy_obj['location'] not in processed_locations:
processed_locations.add(strategy_obj['location'])
start_conversion(strategy_obj, config)
def start_conversion(strategy_obj, config):
print(f"Conversion of {Path(strategy_obj['location']).name} started.")
instance_strategy_updater = StrategyUpdater()
start = time.perf_counter()
instance_strategy_updater.start(config, strategy_obj)
elapsed = time.perf_counter() - start
print(f"Conversion of {Path(strategy_obj['location']).name} took {elapsed:.1f} seconds.")

View File

@ -0,0 +1,255 @@
import shutil
from pathlib import Path
import ast_comments
from freqtrade.constants import Config
class StrategyUpdater:
name_mapping = {
'ticker_interval': 'timeframe',
'buy': 'enter_long',
'sell': 'exit_long',
'buy_tag': 'enter_tag',
'sell_reason': 'exit_reason',
'sell_signal': 'exit_signal',
'custom_sell': 'custom_exit',
'force_sell': 'force_exit',
'emergency_sell': 'emergency_exit',
# Strategy/config settings:
'use_sell_signal': 'use_exit_signal',
'sell_profit_only': 'exit_profit_only',
'sell_profit_offset': 'exit_profit_offset',
'ignore_roi_if_buy_signal': 'ignore_roi_if_entry_signal',
'forcebuy_enable': 'force_entry_enable',
}
function_mapping = {
'populate_buy_trend': 'populate_entry_trend',
'populate_sell_trend': 'populate_exit_trend',
'custom_sell': 'custom_exit',
'check_buy_timeout': 'check_entry_timeout',
'check_sell_timeout': 'check_exit_timeout',
# '': '',
}
# order_time_in_force, order_types, unfilledtimeout
otif_ot_unfilledtimeout = {
'buy': 'entry',
'sell': 'exit',
}
# create a dictionary that maps the old column names to the new ones
rename_dict = {'buy': 'enter_long', 'sell': 'exit_long', 'buy_tag': 'enter_tag'}
def start(self, config: Config, strategy_obj: dict) -> None:
"""
Run strategy updater
It updates a strategy to v3 with the help of the ast-module
:return: None
"""
source_file = strategy_obj['location']
strategies_backup_folder = Path.joinpath(config['user_data_dir'], "strategies_orig_updater")
target_file = Path.joinpath(strategies_backup_folder, strategy_obj['location_rel'])
# read the file
with Path(source_file).open('r') as f:
old_code = f.read()
if not strategies_backup_folder.is_dir():
Path(strategies_backup_folder).mkdir(parents=True, exist_ok=True)
# backup original
# => currently no date after the filename,
# could get overridden pretty fast if this is fired twice!
# The folder is always the same and the file name too (currently).
shutil.copy(source_file, target_file)
# update the code
new_code = self.update_code(old_code)
# write the modified code to the destination folder
with Path(source_file).open('w') as f:
f.write(new_code)
# define the function to update the code
def update_code(self, code):
# parse the code into an AST
tree = ast_comments.parse(code)
# use the AST to update the code
updated_code = self.modify_ast(tree)
# return the modified code without executing it
return updated_code
# function that uses the ast module to update the code
def modify_ast(self, tree): # noqa
# use the visitor to update the names and functions in the AST
NameUpdater().visit(tree)
# first fix the comments, so it understands "\n" properly inside multi line comments.
ast_comments.fix_missing_locations(tree)
ast_comments.increment_lineno(tree, n=1)
# generate the new code from the updated AST
# without indent {} parameters would just be written straight one after the other.
# ast_comments would be amazing since this is the only solution that carries over comments,
# but it does currently not have an unparse function, hopefully in the future ... !
# return ast_comments.unparse(tree)
return ast_comments.unparse(tree)
# Here we go through each respective node, slice, elt, key ... to replace outdated entries.
class NameUpdater(ast_comments.NodeTransformer):
def generic_visit(self, node):
# space is not yet transferred from buy/sell to entry/exit and thereby has to be skipped.
if isinstance(node, ast_comments.keyword):
if node.arg == "space":
return node
# from here on this is the original function.
for field, old_value in ast_comments.iter_fields(node):
if isinstance(old_value, list):
new_values = []
for value in old_value:
if isinstance(value, ast_comments.AST):
value = self.visit(value)
if value is None:
continue
elif not isinstance(value, ast_comments.AST):
new_values.extend(value)
continue
new_values.append(value)
old_value[:] = new_values
elif isinstance(old_value, ast_comments.AST):
new_node = self.visit(old_value)
if new_node is None:
delattr(node, field)
else:
setattr(node, field, new_node)
return node
def visit_Expr(self, node):
if hasattr(node.value, "left") and hasattr(node.value.left, "id"):
node.value.left.id = self.check_dict(StrategyUpdater.name_mapping, node.value.left.id)
self.visit(node.value)
return node
# Renames an element if contained inside a dictionary.
@staticmethod
def check_dict(current_dict: dict, element: str):
if element in current_dict:
element = current_dict[element]
return element
def visit_arguments(self, node):
if isinstance(node.args, list):
for arg in node.args:
arg.arg = self.check_dict(StrategyUpdater.name_mapping, arg.arg)
return node
def visit_Name(self, node):
# if the name is in the mapping, update it
node.id = self.check_dict(StrategyUpdater.name_mapping, node.id)
return node
def visit_Import(self, node):
# do not update the names in import statements
return node
def visit_ImportFrom(self, node):
# if hasattr(node, "module"):
# if node.module == "freqtrade.strategy.hyper":
# node.module = "freqtrade.strategy"
return node
def visit_If(self, node: ast_comments.If):
for child in ast_comments.iter_child_nodes(node):
self.visit(child)
return node
def visit_FunctionDef(self, node):
node.name = self.check_dict(StrategyUpdater.function_mapping, node.name)
self.generic_visit(node)
return node
def visit_Attribute(self, node):
if (
isinstance(node.value, ast_comments.Name)
and node.value.id == 'trade'
and node.attr == 'nr_of_successful_buys'
):
node.attr = 'nr_of_successful_entries'
return node
def visit_ClassDef(self, node):
# check if the class is derived from IStrategy
if any(isinstance(base, ast_comments.Name) and
base.id == 'IStrategy' for base in node.bases):
# check if the INTERFACE_VERSION variable exists
has_interface_version = any(
isinstance(child, ast_comments.Assign) and
isinstance(child.targets[0], ast_comments.Name) and
child.targets[0].id == 'INTERFACE_VERSION'
for child in node.body
)
# if the INTERFACE_VERSION variable does not exist, add it as the first child
if not has_interface_version:
node.body.insert(0, ast_comments.parse('INTERFACE_VERSION = 3').body[0])
# otherwise, update its value to 3
else:
for child in node.body:
if (
isinstance(child, ast_comments.Assign)
and isinstance(child.targets[0], ast_comments.Name)
and child.targets[0].id == 'INTERFACE_VERSION'
):
child.value = ast_comments.parse('3').body[0].value
self.generic_visit(node)
return node
def visit_Subscript(self, node):
if isinstance(node.slice, ast_comments.Constant):
if node.slice.value in StrategyUpdater.rename_dict:
# Replace the slice attributes with the values from rename_dict
node.slice.value = StrategyUpdater.rename_dict[node.slice.value]
if hasattr(node.slice, "elts"):
self.visit_elts(node.slice.elts)
if hasattr(node.slice, "value"):
if hasattr(node.slice.value, "elts"):
self.visit_elts(node.slice.value.elts)
return node
# elts can have elts (technically recursively)
def visit_elts(self, elts):
if isinstance(elts, list):
for elt in elts:
self.visit_elt(elt)
else:
self.visit_elt(elts)
return elts
# sub function again needed since the structure itself is highly flexible ...
def visit_elt(self, elt):
if isinstance(elt, ast_comments.Constant) and elt.value in StrategyUpdater.rename_dict:
elt.value = StrategyUpdater.rename_dict[elt.value]
if hasattr(elt, "elts"):
self.visit_elts(elt.elts)
if hasattr(elt, "args"):
if isinstance(elt.args, ast_comments.arguments):
self.visit_elts(elt.args)
else:
for arg in elt.args:
self.visit_elts(arg)
return elt
def visit_Constant(self, node):
node.value = self.check_dict(StrategyUpdater.otif_ot_unfilledtimeout, node.value)
node.value = self.check_dict(StrategyUpdater.name_mapping, node.value)
return node

View File

@ -55,3 +55,5 @@ schedule==1.1.0
#WS Messages #WS Messages
websockets==10.4 websockets==10.4
janus==1.0.0 janus==1.0.0
ast-comments==1.0.0

View File

@ -14,7 +14,8 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star
start_hyperopt_show, start_install_ui, start_list_data, start_hyperopt_show, start_install_ui, start_list_data,
start_list_exchanges, start_list_markets, start_list_strategies, start_list_exchanges, start_list_markets, start_list_strategies,
start_list_timeframes, start_new_strategy, start_show_trades, start_list_timeframes, start_new_strategy, start_show_trades,
start_test_pairlist, start_trading, start_webserver) start_strategy_update, start_test_pairlist, start_trading,
start_webserver)
from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.db_commands import start_convert_db
from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui,
get_ui_download_url, read_ui_version) get_ui_download_url, read_ui_version)
@ -1546,3 +1547,37 @@ def test_start_convert_db(mocker, fee, tmpdir, caplog):
start_convert_db(pargs) start_convert_db(pargs)
assert db_target_file.is_file() assert db_target_file.is_file()
def test_start_strategy_updater(mocker, tmpdir):
sc_mock = mocker.patch('freqtrade.commands.strategy_utils_commands.start_conversion')
teststrats = Path(__file__).parent.parent / 'strategy/strats'
args = [
"strategy-updater",
"--userdir",
str(tmpdir),
"--strategy-path",
str(teststrats),
]
pargs = get_args(args)
pargs['config'] = None
start_strategy_update(pargs)
# Number of strategies in the test directory
assert sc_mock.call_count == 11
sc_mock.reset_mock()
args = [
"strategy-updater",
"--userdir",
str(tmpdir),
"--strategy-path",
str(teststrats),
"--strategy-list",
"StrategyTestV3",
"StrategyTestV2"
]
pargs = get_args(args)
pargs['config'] = None
start_strategy_update(pargs)
# Number of strategies in the test directory
assert sc_mock.call_count == 2

View File

@ -0,0 +1,214 @@
# pragma pylint: disable=missing-docstring, protected-access, invalid-name
import re
import shutil
import sys
from pathlib import Path
import pytest
from freqtrade.commands.strategy_utils_commands import start_strategy_update
from freqtrade.strategy.strategyupdater import StrategyUpdater
from tests.conftest import get_args
if sys.version_info < (3, 9):
pytest.skip("StrategyUpdater is not compatible with Python 3.8", allow_module_level=True)
def test_strategy_updater_start(tmpdir, capsys) -> None:
# Effective test without mocks.
teststrats = Path(__file__).parent / 'strategy/strats'
tmpdirp = Path(tmpdir) / "strategies"
tmpdirp.mkdir()
shutil.copy(teststrats / 'strategy_test_v2.py', tmpdirp)
old_code = (teststrats / 'strategy_test_v2.py').read_text()
args = [
"strategy-updater",
"--userdir",
str(tmpdir),
"--strategy-list",
"StrategyTestV2"
]
pargs = get_args(args)
pargs['config'] = None
start_strategy_update(pargs)
assert Path(tmpdir / "strategies_orig_updater").exists()
# Backup file exists
assert Path(tmpdir / "strategies_orig_updater" / 'strategy_test_v2.py').exists()
# updated file exists
new_file = Path(tmpdirp / 'strategy_test_v2.py')
assert new_file.exists()
new_code = new_file.read_text()
assert 'INTERFACE_VERSION = 3' in new_code
assert 'INTERFACE_VERSION = 2' in old_code
captured = capsys.readouterr()
assert 'Conversion of strategy_test_v2.py started.' in captured.out
assert re.search(r'Conversion of strategy_test_v2\.py took .* seconds', captured.out)
def test_strategy_updater_methods(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code1 = instance_strategy_updater.update_code("""
class testClass(IStrategy):
def populate_buy_trend():
pass
def populate_sell_trend():
pass
def check_buy_timeout():
pass
def check_sell_timeout():
pass
def custom_sell():
pass
""")
assert "populate_entry_trend" in modified_code1
assert "populate_exit_trend" in modified_code1
assert "check_entry_timeout" in modified_code1
assert "check_exit_timeout" in modified_code1
assert "custom_exit" in modified_code1
assert "INTERFACE_VERSION = 3" in modified_code1
def test_strategy_updater_params(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code2 = instance_strategy_updater.update_code("""
ticker_interval = '15m'
buy_some_parameter = IntParameter(space='buy')
sell_some_parameter = IntParameter(space='sell')
""")
assert "timeframe" in modified_code2
# check for not editing hyperopt spaces
assert "space='buy'" in modified_code2
assert "space='sell'" in modified_code2
def test_strategy_updater_constants(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code3 = instance_strategy_updater.update_code("""
use_sell_signal = True
sell_profit_only = True
sell_profit_offset = True
ignore_roi_if_buy_signal = True
forcebuy_enable = True
""")
assert "use_exit_signal" in modified_code3
assert "exit_profit_only" in modified_code3
assert "exit_profit_offset" in modified_code3
assert "ignore_roi_if_entry_signal" in modified_code3
assert "force_entry_enable" in modified_code3
def test_strategy_updater_df_columns(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code = instance_strategy_updater.update_code("""
dataframe.loc[reduce(lambda x, y: x & y, conditions), ["buy", "buy_tag"]] = (1, "buy_signal_1")
dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1
""")
assert "enter_long" in modified_code
assert "exit_long" in modified_code
assert "enter_tag" in modified_code
def test_strategy_updater_method_params(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code = instance_strategy_updater.update_code("""
def confirm_trade_exit(sell_reason: str):
nr_orders = trade.nr_of_successful_buys
pass
""")
assert "exit_reason" in modified_code
assert "nr_orders = trade.nr_of_successful_entries" in modified_code
def test_strategy_updater_dicts(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code = instance_strategy_updater.update_code("""
order_time_in_force = {
'buy': 'gtc',
'sell': 'ioc'
}
order_types = {
'buy': 'limit',
'sell': 'market',
'stoploss': 'market',
'stoploss_on_exchange': False
}
unfilledtimeout = {
'buy': 1,
'sell': 2
}
""")
assert "'entry': 'gtc'" in modified_code
assert "'exit': 'ioc'" in modified_code
assert "'entry': 'limit'" in modified_code
assert "'exit': 'market'" in modified_code
assert "'entry': 1" in modified_code
assert "'exit': 2" in modified_code
def test_strategy_updater_comparisons(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code = instance_strategy_updater.update_code("""
def confirm_trade_exit(sell_reason):
if (sell_reason == 'stop_loss'):
pass
""")
assert "exit_reason" in modified_code
assert "exit_reason == 'stop_loss'" in modified_code
def test_strategy_updater_strings(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code = instance_strategy_updater.update_code("""
sell_reason == 'sell_signal'
sell_reason == 'force_sell'
sell_reason == 'emergency_sell'
""")
# those tests currently don't work, next in line.
assert "exit_signal" in modified_code
assert "exit_reason" in modified_code
assert "force_exit" in modified_code
assert "emergency_exit" in modified_code
def test_strategy_updater_comments(default_conf, caplog) -> None:
instance_strategy_updater = StrategyUpdater()
modified_code = instance_strategy_updater.update_code("""
# This is the 1st comment
import talib.abstract as ta
# This is the 2nd comment
import freqtrade.vendor.qtpylib.indicators as qtpylib
class someStrategy(IStrategy):
INTERFACE_VERSION = 2
# This is the 3rd comment
# This attribute will be overridden if the config file contains "minimal_roi"
minimal_roi = {
"0": 0.50
}
# This is the 4th comment
stoploss = -0.1
""")
assert "This is the 1st comment" in modified_code
assert "This is the 2nd comment" in modified_code
assert "This is the 3rd comment" in modified_code
assert "INTERFACE_VERSION = 3" in modified_code
# currently still missing:
# Webhook terminology, Telegram notification settings, Strategy/Config settings