From a68445692bfdde1d9407a8d3bc1f82d785cd3def Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Dec 2019 16:08:33 +0100 Subject: [PATCH 01/11] Add first steps for list-strategies --- freqtrade/configuration/arguments.py | 19 ++++++++++++++++--- freqtrade/utils.py | 8 ++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 41c5c3957..5f7bc74f1 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,6 +30,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] +ARGS_LIST_STRATEGIES = ["strategy_path"] + ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] @@ -62,7 +64,8 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperopt_show_no_header"] NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs", - "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit"] + "list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe", + "plot-profit"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] @@ -131,8 +134,9 @@ class Arguments: from freqtrade.utils import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, - start_new_hyperopt, start_new_strategy, - start_list_timeframes, start_test_pairlist, start_trading) + start_list_strategies, start_new_hyperopt, + start_new_strategy, start_list_timeframes, + start_test_pairlist, start_trading) from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit subparsers = self.parser.add_subparsers(dest='command', @@ -185,6 +189,15 @@ class Arguments: build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) + # Add list-strategies subcommand + list_strategies_cmd = subparsers.add_parser( + 'list-strategies', + help='Print available strategies.', + parents=[_common_parser], + ) + list_strategies_cmd.set_defaults(func=start_list_strategies) + self._build_args(optionlist=ARGS_LIST_STRATEGIES, parser=list_strategies_cmd) + # Add list-exchanges subcommand list_exchanges_cmd = subparsers.add_parser( 'list-exchanges', diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 18966c574..c5fc47a74 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -224,6 +224,14 @@ def start_download_data(args: Dict[str, Any]) -> None: f"on exchange {exchange.name}.") +def start_list_strategies(args: Dict[str, Any]) -> None: + """ + Print Strategies available in a folder + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + print(config) + + def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print ticker intervals (timeframes) available on Exchange From eb1040ddb7445a49168792871ab417360bdfa4a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 13:34:37 +0100 Subject: [PATCH 02/11] Convert resolvers to classmethods --- freqtrade/resolvers/exchange_resolver.py | 1 + freqtrade/resolvers/hyperopt_resolver.py | 23 +++++++++------- freqtrade/resolvers/iresolver.py | 34 +++++++++++++----------- freqtrade/resolvers/pairlist_resolver.py | 6 +++-- freqtrade/resolvers/strategy_resolver.py | 12 +++++---- tests/strategy/test_strategy.py | 2 -- 6 files changed, 43 insertions(+), 35 deletions(-) diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index e28a5cf80..2b6a731a9 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -14,6 +14,7 @@ class ExchangeResolver(IResolver): """ This class contains all the logic to load a custom exchange class """ + object_type = Exchange @staticmethod def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 0726b0627..b9c750251 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -20,6 +20,7 @@ class HyperOptResolver(IResolver): """ This class contains all the logic to load custom hyperopt class """ + object_type = IHyperOpt @staticmethod def load_hyperopt(config: Dict) -> IHyperOpt: @@ -59,12 +60,13 @@ class HyperOptResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) + abs_paths = HyperOptResolver.build_search_paths(config, current_path=current_path, + user_subdir=USERPATH_HYPEROPTS, + extra_dir=extra_dir) - hyperopt = IResolver._load_object(paths=abs_paths, object_type=IHyperOpt, - object_name=hyperopt_name, kwargs={'config': config}) + hyperopt = HyperOptResolver._load_object(paths=abs_paths, + object_name=hyperopt_name, + kwargs={'config': config}) if hyperopt: return hyperopt raise OperationalException( @@ -77,6 +79,7 @@ class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class """ + object_type = IHyperOptLoss @staticmethod def load_hyperoptloss(config: Dict) -> IHyperOptLoss: @@ -113,12 +116,12 @@ class HyperOptLossResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) + abs_paths = HyperOptLossResolver.build_search_paths(config, current_path=current_path, + user_subdir=USERPATH_HYPEROPTS, + extra_dir=extra_dir) - hyperoptloss = IResolver._load_object(paths=abs_paths, object_type=IHyperOptLoss, - object_name=hyper_loss_name) + hyperoptloss = HyperOptLossResolver._load_object(paths=abs_paths, + object_name=hyper_loss_name) if hyperoptloss: return hyperoptloss diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 0b986debb..11937c1da 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -7,7 +7,7 @@ import importlib.util import inspect import logging from pathlib import Path -from typing import Any, List, Optional, Tuple, Union, Generator +from typing import Any, Generator, List, Optional, Tuple, Type, Union logger = logging.getLogger(__name__) @@ -16,6 +16,8 @@ class IResolver: """ This class contains all the logic to load custom classes """ + # Childclasses need to override this + object_type: Type[Any] @staticmethod def build_search_paths(config, current_path: Path, user_subdir: Optional[str] = None, @@ -32,12 +34,11 @@ class IResolver: return abs_paths - @staticmethod - def _get_valid_object(object_type, module_path: Path, + @classmethod + def _get_valid_object(cls, module_path: Path, object_name: str) -> Generator[Any, None, None]: """ Generator returning objects with matching object_type and object_name in the path given. - :param object_type: object_type (class) :param module_path: absolute path to the module :param object_name: Class name of the object :return: generator containing matching objects @@ -55,19 +56,21 @@ class IResolver: valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) - if object_name == name and object_type in obj.__bases__ + if object_name == name and cls.object_type in obj.__bases__ ) return valid_objects_gen - @staticmethod - def _search_object(directory: Path, object_type, object_name: str, + @classmethod + def _search_object(cls, directory: Path, object_name: str, kwargs: dict = {}) -> Union[Tuple[Any, Path], Tuple[None, None]]: """ Search for the objectname in the given directory :param directory: relative or absolute directory path + :param object_name: ClassName of the object to load :return: object instance """ - logger.debug("Searching for %s %s in '%s'", object_type.__name__, object_name, directory) + logger.debug("Searching for %s %s in '%s'", + cls.object_type.__name__, object_name, directory) for entry in directory.iterdir(): # Only consider python files if not str(entry).endswith('.py'): @@ -75,14 +78,14 @@ class IResolver: continue module_path = entry.resolve() - obj = next(IResolver._get_valid_object(object_type, module_path, object_name), None) + obj = next(cls._get_valid_object(module_path, object_name), None) if obj: return (obj(**kwargs), module_path) return (None, None) - @staticmethod - def _load_object(paths: List[Path], object_type, object_name: str, + @classmethod + def _load_object(cls, paths: List[Path], object_name: str, kwargs: dict = {}) -> Optional[Any]: """ Try to load object from path list. @@ -90,13 +93,12 @@ class IResolver: for _path in paths: try: - (module, module_path) = IResolver._search_object(directory=_path, - object_type=object_type, - object_name=object_name, - kwargs=kwargs) + (module, module_path) = cls._search_object(directory=_path, + object_name=object_name, + kwargs=kwargs) if module: logger.info( - f"Using resolved {object_type.__name__.lower()[1:]} {object_name} " + f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} " f"from '{module_path}'...") return module except FileNotFoundError: diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 611660ff4..00ebc03aa 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -17,6 +17,7 @@ class PairListResolver(IResolver): """ This class contains all the logic to load custom PairList class """ + object_type = IPairList @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, @@ -53,8 +54,9 @@ class PairListResolver(IResolver): abs_paths = IResolver.build_search_paths(config, current_path=current_path, user_subdir=None, extra_dir=None) - pairlist = IResolver._load_object(paths=abs_paths, object_type=IPairList, - object_name=pairlist_name, kwargs=kwargs) + pairlist = PairListResolver._load_object(paths=abs_paths, + object_name=pairlist_name, + kwargs=kwargs) if pairlist: return pairlist raise OperationalException( diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 6d3fe5ff9..654103377 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -22,6 +22,7 @@ class StrategyResolver(IResolver): """ This class contains the logic to load custom strategy class """ + object_type = IStrategy @staticmethod def load_strategy(config: Optional[Dict] = None) -> IStrategy: @@ -134,9 +135,9 @@ class StrategyResolver(IResolver): """ current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=constants.USERPATH_STRATEGY, - extra_dir=extra_dir) + abs_paths = StrategyResolver.build_search_paths(config, current_path=current_path, + user_subdir=constants.USERPATH_STRATEGY, + extra_dir=extra_dir) if ":" in strategy_name: logger.info("loading base64 encoded strategy") @@ -154,8 +155,9 @@ class StrategyResolver(IResolver): # register temp path with the bot abs_paths.insert(0, temp.resolve()) - strategy = IResolver._load_object(paths=abs_paths, object_type=IStrategy, - object_name=strategy_name, kwargs={'config': config}) + strategy = StrategyResolver._load_object(paths=abs_paths, + object_name=strategy_name, + kwargs={'config': config}) if strategy: strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index ce7ac1741..dba816621 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -20,7 +20,6 @@ def test_search_strategy(): s, _ = StrategyResolver._search_object( directory=default_location, - object_type=IStrategy, kwargs={'config': default_config}, object_name='DefaultStrategy' ) @@ -28,7 +27,6 @@ def test_search_strategy(): s, _ = StrategyResolver._search_object( directory=default_location, - object_type=IStrategy, kwargs={'config': default_config}, object_name='NotFoundStrategy' ) From 25e6d6a7bfdea9bd74b6f06375c4e11df2f903b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 13:54:46 +0100 Subject: [PATCH 03/11] Combine load_object methods into one --- freqtrade/resolvers/hyperopt_resolver.py | 70 +++++------------------- freqtrade/resolvers/iresolver.py | 36 +++++++++++- freqtrade/resolvers/pairlist_resolver.py | 42 ++++---------- freqtrade/resolvers/strategy_resolver.py | 16 ++++-- tests/optimize/test_hyperopt.py | 4 +- 5 files changed, 68 insertions(+), 100 deletions(-) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index b9c750251..c26fd09f2 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -5,7 +5,7 @@ This module load custom hyperopt """ import logging from pathlib import Path -from typing import Optional, Dict +from typing import Dict from freqtrade import OperationalException from freqtrade.constants import DEFAULT_HYPEROPT_LOSS, USERPATH_HYPEROPTS @@ -21,6 +21,9 @@ class HyperOptResolver(IResolver): This class contains all the logic to load custom hyperopt class """ object_type = IHyperOpt + object_type_str = "Hyperopt" + user_subdir = USERPATH_HYPEROPTS + initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() @staticmethod def load_hyperopt(config: Dict) -> IHyperOpt: @@ -34,8 +37,9 @@ class HyperOptResolver(IResolver): hyperopt_name = config['hyperopt'] - hyperopt = HyperOptResolver._load_hyperopt(hyperopt_name, config, - extra_dir=config.get('hyperopt_path')) + hyperopt = HyperOptResolver.load_object(hyperopt_name, config, + kwargs={'config': config}, + extra_dir=config.get('hyperopt_path')) if not hasattr(hyperopt, 'populate_indicators'): logger.warning("Hyperopt class does not provide populate_indicators() method. " @@ -48,38 +52,15 @@ class HyperOptResolver(IResolver): "Using populate_sell_trend from the strategy.") return hyperopt - @staticmethod - def _load_hyperopt( - hyperopt_name: str, config: Dict, extra_dir: Optional[str] = None) -> IHyperOpt: - """ - Search and loads the specified hyperopt. - :param hyperopt_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given hyperopt - :return: HyperOpt instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - - abs_paths = HyperOptResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) - - hyperopt = HyperOptResolver._load_object(paths=abs_paths, - object_name=hyperopt_name, - kwargs={'config': config}) - if hyperopt: - return hyperopt - raise OperationalException( - f"Impossible to load Hyperopt '{hyperopt_name}'. This class does not exist " - "or contains Python code errors." - ) - class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class """ object_type = IHyperOptLoss + object_type_str = "HyperoptLoss" + user_subdir = USERPATH_HYPEROPTS + initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() @staticmethod def load_hyperoptloss(config: Dict) -> IHyperOptLoss: @@ -92,8 +73,9 @@ class HyperOptLossResolver(IResolver): # default hyperopt loss hyperoptloss_name = config.get('hyperopt_loss') or DEFAULT_HYPEROPT_LOSS - hyperoptloss = HyperOptLossResolver._load_hyperoptloss( - hyperoptloss_name, config, extra_dir=config.get('hyperopt_path')) + hyperoptloss = HyperOptLossResolver.load_object(hyperoptloss_name, + config, kwargs={}, + extra_dir=config.get('hyperopt_path')) # Assign ticker_interval to be used in hyperopt hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) @@ -103,29 +85,3 @@ class HyperOptLossResolver(IResolver): f"Found HyperoptLoss class {hyperoptloss_name} does not " "implement `hyperopt_loss_function`.") return hyperoptloss - - @staticmethod - def _load_hyperoptloss(hyper_loss_name: str, config: Dict, - extra_dir: Optional[str] = None) -> IHyperOptLoss: - """ - Search and loads the specified hyperopt loss class. - :param hyper_loss_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given hyperopt - :return: HyperOptLoss instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - - abs_paths = HyperOptLossResolver.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, - extra_dir=extra_dir) - - hyperoptloss = HyperOptLossResolver._load_object(paths=abs_paths, - object_name=hyper_loss_name) - if hyperoptloss: - return hyperoptloss - - raise OperationalException( - f"Impossible to load HyperoptLoss '{hyper_loss_name}'. This class does not exist " - "or contains Python code errors." - ) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 11937c1da..0101e37a3 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -9,6 +9,8 @@ import logging from pathlib import Path from typing import Any, Generator, List, Optional, Tuple, Type, Union +from freqtrade import OperationalException + logger = logging.getLogger(__name__) @@ -18,12 +20,15 @@ class IResolver: """ # Childclasses need to override this object_type: Type[Any] + object_type_str: str + user_subdir: Optional[str] = None + initial_search_path: Path - @staticmethod - def build_search_paths(config, current_path: Path, user_subdir: Optional[str] = None, + @classmethod + def build_search_paths(cls, config, user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: - abs_paths: List[Path] = [current_path] + abs_paths: List[Path] = [cls.initial_search_path] if user_subdir: abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) @@ -105,3 +110,28 @@ class IResolver: logger.warning('Path "%s" does not exist.', _path.resolve()) return None + + @classmethod + def load_object(cls, object_name: str, config: dict, kwargs: dict, + extra_dir: Optional[str] = None) -> Any: + """ + Search and loads the specified object as configured in hte child class. + :param objectname: name of the module to import + :param config: configuration dictionary + :param extra_dir: additional directory to search for the given pairlist + :raises: OperationalException if the class is invalid or does not exist. + :return: Object instance or None + """ + + abs_paths = cls.build_search_paths(config, + user_subdir=cls.user_subdir, + extra_dir=extra_dir) + + pairlist = cls._load_object(paths=abs_paths, object_name=object_name, + kwargs=kwargs) + if pairlist: + return pairlist + raise OperationalException( + f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist " + "or contains Python code errors." + ) diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 00ebc03aa..77db74084 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -6,7 +6,6 @@ This module load custom pairlists import logging from pathlib import Path -from freqtrade import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import IResolver @@ -18,6 +17,9 @@ class PairListResolver(IResolver): This class contains all the logic to load custom PairList class """ object_type = IPairList + object_type_str = "Pairlist" + user_subdir = None + initial_search_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, @@ -32,34 +34,10 @@ class PairListResolver(IResolver): :param pairlist_pos: Position of the pairlist in the list of pairlists :return: initialized Pairlist class """ - - return PairListResolver._load_pairlist(pairlist_name, config, - kwargs={'exchange': exchange, - 'pairlistmanager': pairlistmanager, - 'config': config, - 'pairlistconfig': pairlistconfig, - 'pairlist_pos': pairlist_pos}) - - @staticmethod - def _load_pairlist(pairlist_name: str, config: dict, kwargs: dict) -> IPairList: - """ - Search and loads the specified pairlist. - :param pairlist_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given pairlist - :return: PairList instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() - - abs_paths = IResolver.build_search_paths(config, current_path=current_path, - user_subdir=None, extra_dir=None) - - pairlist = PairListResolver._load_object(paths=abs_paths, - object_name=pairlist_name, - kwargs=kwargs) - if pairlist: - return pairlist - raise OperationalException( - f"Impossible to load Pairlist '{pairlist_name}'. This class does not exist " - "or contains Python code errors." - ) + return PairListResolver.load_object(pairlist_name, config, + kwargs={'exchange': exchange, + 'pairlistmanager': pairlistmanager, + 'config': config, + 'pairlistconfig': pairlistconfig, + 'pairlist_pos': pairlist_pos}, + ) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 654103377..4fd5c586a 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -11,7 +11,9 @@ from inspect import getfullargspec from pathlib import Path from typing import Dict, Optional -from freqtrade import constants, OperationalException +from freqtrade import OperationalException +from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, + USERPATH_STRATEGY) from freqtrade.resolvers import IResolver from freqtrade.strategy.interface import IStrategy @@ -23,6 +25,9 @@ class StrategyResolver(IResolver): This class contains the logic to load custom strategy class """ object_type = IStrategy + object_type_str = "Strategy" + user_subdir = USERPATH_STRATEGY + initial_search_path = Path(__file__).parent.parent.joinpath('strategy').resolve() @staticmethod def load_strategy(config: Optional[Dict] = None) -> IStrategy: @@ -115,11 +120,11 @@ class StrategyResolver(IResolver): @staticmethod def _strategy_sanity_validations(strategy): - if not all(k in strategy.order_types for k in constants.REQUIRED_ORDERTYPES): + if not all(k in strategy.order_types for k in REQUIRED_ORDERTYPES): raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-types mapping is incomplete.") - if not all(k in strategy.order_time_in_force for k in constants.REQUIRED_ORDERTIF): + if not all(k in strategy.order_time_in_force for k in REQUIRED_ORDERTIF): raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-time-in-force mapping is incomplete.") @@ -133,10 +138,9 @@ class StrategyResolver(IResolver): :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ - current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() - abs_paths = StrategyResolver.build_search_paths(config, current_path=current_path, - user_subdir=constants.USERPATH_STRATEGY, + abs_paths = StrategyResolver.build_search_paths(config, + user_subdir=USERPATH_STRATEGY, extra_dir=extra_dir) if ":" in strategy_name: diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 9c6e73c53..fb492be35 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -159,7 +159,7 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: delattr(hyperopt, 'populate_buy_trend') delattr(hyperopt, 'populate_sell_trend') mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt', + 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver.load_object', MagicMock(return_value=hyperopt(default_conf)) ) default_conf.update({'hyperopt': 'DefaultHyperOpt'}) @@ -195,7 +195,7 @@ def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None: hl = DefaultHyperOptLoss mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver._load_hyperoptloss', + 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', MagicMock(return_value=hl) ) x = HyperOptLossResolver.load_hyperoptloss(default_conf) From 5a11ca86bb69b1619cbba78cce05a205f669a67d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 14:01:28 +0100 Subject: [PATCH 04/11] Move instanciation out of search_object --- freqtrade/resolvers/iresolver.py | 13 ++++++------- tests/strategy/test_strategy.py | 5 +---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 0101e37a3..bbdc8ca91 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -66,13 +66,13 @@ class IResolver: return valid_objects_gen @classmethod - def _search_object(cls, directory: Path, object_name: str, - kwargs: dict = {}) -> Union[Tuple[Any, Path], Tuple[None, None]]: + def _search_object(cls, directory: Path, object_name: str + ) -> Union[Tuple[Any, Path], Tuple[None, None]]: """ Search for the objectname in the given directory :param directory: relative or absolute directory path :param object_name: ClassName of the object to load - :return: object instance + :return: object class """ logger.debug("Searching for %s %s in '%s'", cls.object_type.__name__, object_name, directory) @@ -86,7 +86,7 @@ class IResolver: obj = next(cls._get_valid_object(module_path, object_name), None) if obj: - return (obj(**kwargs), module_path) + return (obj, module_path) return (None, None) @classmethod @@ -99,13 +99,12 @@ class IResolver: for _path in paths: try: (module, module_path) = cls._search_object(directory=_path, - object_name=object_name, - kwargs=kwargs) + object_name=object_name) if module: logger.info( f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} " f"from '{module_path}'...") - return module + return module(**kwargs) except FileNotFoundError: logger.warning('Path "%s" does not exist.', _path.resolve()) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index dba816621..7085223c5 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -15,19 +15,16 @@ from tests.conftest import log_has, log_has_re def test_search_strategy(): - default_config = {} default_location = Path(__file__).parent.parent.joinpath('strategy').resolve() s, _ = StrategyResolver._search_object( directory=default_location, - kwargs={'config': default_config}, object_name='DefaultStrategy' ) - assert isinstance(s, IStrategy) + assert issubclass(s, IStrategy) s, _ = StrategyResolver._search_object( directory=default_location, - kwargs={'config': default_config}, object_name='NotFoundStrategy' ) assert s is None From 2ab989e274c5bfb278e8b6397380b1714d806a80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:28:35 +0100 Subject: [PATCH 05/11] Cleanup some code and add option --- freqtrade/configuration/arguments.py | 2 +- freqtrade/resolvers/iresolver.py | 35 +++++++++++++++++++++++----- freqtrade/utils.py | 14 +++++++++-- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 5f7bc74f1..b2197619d 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,7 +30,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] -ARGS_LIST_STRATEGIES = ["strategy_path"] +ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column"] ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index bbdc8ca91..01ecbcb84 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -7,7 +7,7 @@ import importlib.util import inspect import logging from pathlib import Path -from typing import Any, Generator, List, Optional, Tuple, Type, Union +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, Union from freqtrade import OperationalException @@ -41,7 +41,7 @@ class IResolver: @classmethod def _get_valid_object(cls, module_path: Path, - object_name: str) -> Generator[Any, None, None]: + object_name: Optional[str]) -> Generator[Any, None, None]: """ Generator returning objects with matching object_type and object_name in the path given. :param module_path: absolute path to the module @@ -51,7 +51,7 @@ class IResolver: # Generate spec based on absolute path # Pass object_name as first argument to have logging print a reasonable name. - spec = importlib.util.spec_from_file_location(object_name, str(module_path)) + spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints @@ -61,7 +61,7 @@ class IResolver: valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) - if object_name == name and cls.object_type in obj.__bases__ + if (object_name is None or object_name == name) and cls.object_type in obj.__bases__ ) return valid_objects_gen @@ -74,8 +74,7 @@ class IResolver: :param object_name: ClassName of the object to load :return: object class """ - logger.debug("Searching for %s %s in '%s'", - cls.object_type.__name__, object_name, directory) + logger.debug(f"Searching for {cls.object_type.__name__} {object_name} in '{directory}'") for entry in directory.iterdir(): # Only consider python files if not str(entry).endswith('.py'): @@ -134,3 +133,27 @@ class IResolver: f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist " "or contains Python code errors." ) + + @classmethod + def search_all_objects(cls, directory: Path) -> List[Dict[str, Any]]: + """ + Searches a directory for valid objects + :param directory: Path to search + :return: List of dicts containing 'name', 'class' and 'location' entires + """ + logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") + objects = [] + for entry in directory.iterdir(): + # Only consider python files + if not str(entry).endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + module_path = entry.resolve() + logger.info(f"Path {module_path}") + for obj in cls._get_valid_object(module_path, object_name=None): + objects.append( + {'name': obj.__name__, + 'class': obj, + 'location': entry, + }) + return objects diff --git a/freqtrade/utils.py b/freqtrade/utils.py index c5fc47a74..06a62172a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -23,7 +23,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, from freqtrade.exchange import (available_exchanges, ccxt_exchanges, market_is_active, symbol_is_pair) from freqtrade.misc import plural, render_template -from freqtrade.resolvers import ExchangeResolver +from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -229,7 +229,17 @@ def start_list_strategies(args: Dict[str, Any]) -> None: Print Strategies available in a folder """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - print(config) + + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) + strategies = StrategyResolver.search_all_objects(directory) + # Sort alphabetically + strategies = sorted(strategies, key=lambda x: x['name']) + strats_to_print = [{'name': s['name'], 'location': s['location']} for s in strategies] + + if args['print_one_column']: + print('\n'.join([s['name'] for s in strategies])) + else: + print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) def start_list_timeframes(args: Dict[str, Any]) -> None: From 27b86170778c70dd4a83d2c02d06a12452fb1783 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:35:38 +0100 Subject: [PATCH 06/11] Add tests --- tests/strategy/test_strategy.py | 8 +++++++ tests/test_utils.py | 42 +++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 7085223c5..10b9f3466 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -30,6 +30,14 @@ def test_search_strategy(): assert s is None +def test_search_all_strategies(): + directory = Path(__file__).parent + strategies = StrategyResolver.search_all_objects(directory) + assert isinstance(strategies, list) + assert len(strategies) == 3 + assert isinstance(strategies[0], dict) + + def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', 'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates') diff --git a/tests/test_utils.py b/tests/test_utils.py index 40ca9ac02..b8be1ae61 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,11 +7,12 @@ import pytest from freqtrade import OperationalException from freqtrade.state import RunMode from freqtrade.utils import (setup_utils_configuration, start_create_userdir, - start_download_data, start_list_exchanges, - start_list_markets, start_list_timeframes, - start_new_hyperopt, start_new_strategy, - start_test_pairlist, start_trading, - start_hyperopt_list, start_hyperopt_show) + start_download_data, start_hyperopt_list, + start_hyperopt_show, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_hyperopt, + start_new_strategy, start_test_pairlist, + start_trading) from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -630,6 +631,37 @@ def test_download_data_trades(mocker, caplog): assert convert_mock.call_count == 1 +def test_start_list_strategies(mocker, caplog, capsys): + + args = [ + "list-strategies", + "--strategy-path", + str(Path(__file__).parent / "strategy"), + "-1" + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_strategies(pargs) + captured = capsys.readouterr() + assert "TestStrategyLegacy" in captured.out + assert "strategy/legacy_strategy.py" not in captured.out + assert "DefaultStrategy" in captured.out + + # Test regular output + args = [ + "list-strategies", + "--strategy-path", + str(Path(__file__).parent / "strategy"), + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_strategies(pargs) + captured = capsys.readouterr() + assert "TestStrategyLegacy" in captured.out + assert "strategy/legacy_strategy.py" in captured.out + assert "DefaultStrategy" in captured.out + + def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, capsys): mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), From 66f9ece0619554f072fbc20c251b35dd6bbc6f33 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:35:53 +0100 Subject: [PATCH 07/11] Add documentation for strategy-list --- docs/utils.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/utils.md b/docs/utils.md index a9fbfc7d5..6eb37a386 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -108,6 +108,44 @@ With custom user directory freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt ``` +## List Strategies + +Use the `list-strategies` subcommand to see all strategies in one particular folder. + +``` +freqtrade list-strategies --help +usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--strategy-path PATH] [-1] + +optional arguments: + -h, --help show this help message and exit + --strategy-path PATH Specify additional strategy lookup path. + -1, --one-column Print output in one column. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` +Example: search default strategy folder within userdir + +``` bash +freqtrade list-strategies --user-data ~/.freqtrade/ +``` + +Example: search dedicated strategy path + +``` bash +freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ +``` + ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. From 402c761a231edffe017ba93fdf821637e5ddab40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:40:49 +0100 Subject: [PATCH 08/11] Change loglevel of Path output to debug --- docs/utils.md | 5 ++++- freqtrade/resolvers/iresolver.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 6eb37a386..f7501ae9d 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -132,8 +132,11 @@ Common arguments: Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. - ``` + +!!! Warning + Using this command will try to load all python files from a folder. This can be a security risk if untrusted files reside in this folder, since all module-level code is executed. + Example: search default strategy folder within userdir ``` bash diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 01ecbcb84..e3c0d1ad0 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -149,7 +149,7 @@ class IResolver: logger.debug('Ignoring %s', entry) continue module_path = entry.resolve() - logger.info(f"Path {module_path}") + logger.debug(f"Path {module_path}") for obj in cls._get_valid_object(module_path, object_name=None): objects.append( {'name': obj.__name__, From ad75048678796a23e4dcf4a82464628a2860d6b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2019 15:53:40 +0100 Subject: [PATCH 09/11] Fix testing with path in windows --- tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index b8be1ae61..185425efc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -644,7 +644,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert "strategy/legacy_strategy.py" not in captured.out + assert str(Path("strategy/legacy_strategy.py")) not in captured.out assert "DefaultStrategy" in captured.out # Test regular output @@ -658,7 +658,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert "strategy/legacy_strategy.py" in captured.out + assert str(Path("strategy/legacy_strategy.py")) in captured.out assert "DefaultStrategy" in captured.out From fc98cf00372b8b5f0d8e9cdd6ea8dc3b182f3f4a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 06:25:45 +0100 Subject: [PATCH 10/11] Address PR feedback - change output to show Filename only --- docs/utils.md | 8 ++++---- freqtrade/utils.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index f7501ae9d..18deeac54 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -110,7 +110,7 @@ freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt ## List Strategies -Use the `list-strategies` subcommand to see all strategies in one particular folder. +Use the `list-strategies` subcommand to see all strategies in one particular directory. ``` freqtrade list-strategies --help @@ -135,12 +135,12 @@ Common arguments: ``` !!! Warning - Using this command will try to load all python files from a folder. This can be a security risk if untrusted files reside in this folder, since all module-level code is executed. + Using this command will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. -Example: search default strategy folder within userdir +Example: search default strategy directory within userdir ``` bash -freqtrade list-strategies --user-data ~/.freqtrade/ +freqtrade list-strategies --userdir ~/.freqtrade/ ``` Example: search dedicated strategy path diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 06a62172a..f6e251154 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -226,7 +226,7 @@ def start_download_data(args: Dict[str, Any]) -> None: def start_list_strategies(args: Dict[str, Any]) -> None: """ - Print Strategies available in a folder + Print Strategies available in a directory """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -234,7 +234,7 @@ def start_list_strategies(args: Dict[str, Any]) -> None: strategies = StrategyResolver.search_all_objects(directory) # Sort alphabetically strategies = sorted(strategies, key=lambda x: x['name']) - strats_to_print = [{'name': s['name'], 'location': s['location']} for s in strategies] + strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] if args['print_one_column']: print('\n'.join([s['name'] for s in strategies])) From b2fb28453f460f89bfaef57a2ac5edae90749585 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 06:39:25 +0100 Subject: [PATCH 11/11] Fix tests after changing output --- tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 185425efc..4cf7b5f23 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -644,7 +644,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert str(Path("strategy/legacy_strategy.py")) not in captured.out + assert "legacy_strategy.py" not in captured.out assert "DefaultStrategy" in captured.out # Test regular output @@ -658,7 +658,7 @@ def test_start_list_strategies(mocker, caplog, capsys): start_list_strategies(pargs) captured = capsys.readouterr() assert "TestStrategyLegacy" in captured.out - assert str(Path("strategy/legacy_strategy.py")) in captured.out + assert "legacy_strategy.py" in captured.out assert "DefaultStrategy" in captured.out