Merge pull request #6558 from samgermain/recursive-strategy-folder

Recursively search subdirectories in config['user_data_dir']/strategies for a strategy

config_examples/config_full.example.json

@@ -182,6 +182,7 @@
     "disable_dataframe_checks": false,
     "strategy": "SampleStrategy",
     "strategy_path": "user_data/strategies/",
+    "recursive_strategy_search": false,
     "add_config_files": [],
     "dataformat_ohlcv": "json",
     "dataformat_trades": "jsongz"

docs/configuration.md

@@ -173,6 +173,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `order_types` | Configure order-types depending on the action (`"entry"`, `"exit"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Dict
 | `order_time_in_force` | Configure time in force for entry and exit orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Dict
 | `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price. <br>*Defaults to `0.02` 2%).*<br> **Datatype:** Positive float
+| `recursive_strategy_search` | Set to `true` to recursively search sub-directories inside `user_data/strategies` for a strategy. <br> **Datatype:** Boolean
 | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). <br> **Datatype:** String
 | `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.<br> **Datatype:** Boolean
 | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String

freqtrade/commands/arguments.py

@@ -12,7 +12,7 @@ from freqtrade.constants import DEFAULT_CONFIG
 
 ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"]
 
-ARGS_STRATEGY = ["strategy", "strategy_path"]
+ARGS_STRATEGY = ["strategy", "strategy_path", "recursive_strategy_search"]
 
 ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
 
@@ -37,7 +37,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
 
 ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 
-ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized"]
+ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized",
+                        "recursive_strategy_search"]
 
 ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"]
 

freqtrade/commands/cli_options.py

@@ -83,6 +83,11 @@ AVAILABLE_CLI_OPTIONS = {
         help='Reset sample files to their original state.',
         action='store_true',
     ),
+    "recursive_strategy_search": Arg(
+        '--recursive-strategy-search',
+        help='Recursively search for a strategy in the strategies folder.',
+        action='store_true',
+    ),
     # Main options
     "strategy": Arg(
         '-s', '--strategy',

freqtrade/commands/list_commands.py

@@ -41,7 +41,7 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
         print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason']))
 
 
-def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
+def _print_objs_tabular(objs: List, print_colorized: bool, base_dir: Path) -> None:
     if print_colorized:
         colorama_init(autoreset=True)
         red = Fore.RED
@@ -55,7 +55,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
     names = [s['name'] for s in objs]
     objs_to_print = [{
         'name': s['name'] if s['name'] else "--",
-        'location': s['location'].name,
+        'location': s['location'].relative_to(base_dir),
         'status': (red + "LOAD FAILED" + reset if s['class'] is None
                    else "OK" if names.count(s['name']) == 1
                    else yellow + "DUPLICATE NAME" + reset)
@@ -77,7 +77,8 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
-    strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column'])
+    strategy_objs = StrategyResolver.search_all_objects(
+        directory, not args['print_one_column'], config.get('recursive_strategy_search', False))
     # Sort alphabetically
     strategy_objs = sorted(strategy_objs, key=lambda x: x['name'])
     for obj in strategy_objs:
@@ -89,7 +90,7 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
     if args['print_one_column']:
         print('\n'.join([s['name'] for s in strategy_objs]))
     else:
-        _print_objs_tabular(strategy_objs, config.get('print_colorized', False))
+        _print_objs_tabular(strategy_objs, config.get('print_colorized', False), directory)
 
 
 def start_list_timeframes(args: Dict[str, Any]) -> None:

freqtrade/configuration/configuration.py

@@ -248,6 +248,12 @@ class Configuration:
         self._args_to_config(config, argname='strategy_list',
                              logstring='Using strategy list of {} strategies', logfun=len)
 
+        self._args_to_config(
+            config,
+            argname='recursive_strategy_search',
+            logstring='Recursively searching for a strategy in the strategies folder.',
+        )
+
         self._args_to_config(config, argname='timeframe',
                              logstring='Overriding timeframe with Command line argument')
 

freqtrade/optimize/hyperopt_tools.py

@@ -41,7 +41,8 @@ class HyperoptTools():
         """
         from freqtrade.resolvers.strategy_resolver import StrategyResolver
         directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
-        strategy_objs = StrategyResolver.search_all_objects(directory, False)
+        strategy_objs = StrategyResolver.search_all_objects(
+            directory, False, config.get('recursive_strategy_search', False))
         strategies = [s for s in strategy_objs if s['name'] == strategy_name]
         if strategies:
             strategy = strategies[0]

freqtrade/resolvers/iresolver.py

@@ -44,7 +44,7 @@ class IResolver:
 
     @classmethod
     def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None,
-                           extra_dir: Optional[str] = None) -> List[Path]:
+                           extra_dirs: List[str] = []) -> List[Path]:
 
         abs_paths: List[Path] = []
         if cls.initial_search_path:
@@ -53,9 +53,9 @@ class IResolver:
         if user_subdir:
             abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir))
 
-        if extra_dir:
+        # Add extra directory to the top of the search paths
-            # Add extra directory to the top of the search paths
+        for dir in extra_dirs:
-            abs_paths.insert(0, Path(extra_dir).resolve())
+            abs_paths.insert(0, Path(dir).resolve())
 
         return abs_paths
 
@@ -164,9 +164,13 @@ class IResolver:
         :return: Object instance or None
         """
 
+        extra_dirs: List[str] = []
+        if extra_dir:
+            extra_dirs.append(extra_dir)
+
         abs_paths = cls.build_search_paths(config,
                                            user_subdir=cls.user_subdir,
-                                           extra_dir=extra_dir)
+                                           extra_dirs=extra_dirs)
 
         found_object = cls._load_object(paths=abs_paths, object_name=object_name,
                                         kwargs=kwargs)
@@ -178,18 +182,25 @@ class IResolver:
         )
 
     @classmethod
-    def search_all_objects(cls, directory: Path,
+    def search_all_objects(cls, directory: Path, enum_failed: bool,
-                           enum_failed: bool) -> List[Dict[str, Any]]:
+                           recursive: bool = False) -> List[Dict[str, Any]]:
         """
         Searches a directory for valid objects
         :param directory: Path to search
         :param enum_failed: If True, will return None for modules which fail.
             Otherwise, failing modules are skipped.
+        :param recursive: Recursively walk directory tree searching for strategies
         :return: List of dicts containing 'name', 'class' and 'location' entries
         """
         logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'")
         objects = []
         for entry in directory.iterdir():
+            if (
+                recursive and entry.is_dir()
+                and not entry.name.startswith('__')
+                and not entry.name.startswith('.')
+            ):
+                objects.extend(cls.search_all_objects(entry, enum_failed, recursive=recursive))
             # Only consider python files
             if entry.suffix != '.py':
                 logger.debug('Ignoring %s', entry)

freqtrade/resolvers/strategy_resolver.py

@@ -7,8 +7,9 @@ import logging
 import tempfile
 from base64 import urlsafe_b64decode
 from inspect import getfullargspec
+from os import walk
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from freqtrade.configuration.config_validation import validate_migrated_strategy_settings
 from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES
@@ -237,10 +238,19 @@ class StrategyResolver(IResolver):
         :param extra_dir: additional directory to search for the given strategy
         :return: Strategy instance or None
         """
+        if config.get('recursive_strategy_search', False):
+            extra_dirs: List[str] = [
+                path[0] for path in walk(f"{config['user_data_dir']}/{USERPATH_STRATEGIES}")
+            ]  # sub-directories
+        else:
+            extra_dirs = []
+
+        if extra_dir:
+            extra_dirs.append(extra_dir)
 
         abs_paths = StrategyResolver.build_search_paths(config,
                                                         user_subdir=USERPATH_STRATEGIES,
-                                                        extra_dir=extra_dir)
+                                                        extra_dirs=extra_dirs)
 
         if ":" in strategy_name:
             logger.info("loading base64 encoded strategy")

freqtrade/rpc/api_server/api_v1.py

@@ -253,7 +253,8 @@ def list_strategies(config=Depends(get_config)):
     directory = Path(config.get(
         'strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
     from freqtrade.resolvers.strategy_resolver import StrategyResolver
-    strategies = StrategyResolver.search_all_objects(directory, False)
+    strategies = StrategyResolver.search_all_objects(
+        directory, False, config.get('recursive_strategy_search', False))
     strategies = sorted(strategies, key=lambda x: x['name'])
 
     return {'strategies': [x['name'] for x in strategies]}

tests/commands/test_commands.py

@@ -847,7 +847,7 @@ def test_start_convert_trades(mocker, caplog):
     assert convert_mock.call_count == 1
 
 
-def test_start_list_strategies(mocker, caplog, capsys):
+def test_start_list_strategies(capsys):
 
     args = [
         "list-strategies",
@@ -892,6 +892,26 @@ def test_start_list_strategies(mocker, caplog, capsys):
     assert "legacy_strategy_v1.py" in captured.out
     assert CURRENT_TEST_STRATEGY in captured.out
     assert "LOAD FAILED" in captured.out
+    # Recursive
+    assert "TestStrategyNoImplements" not in captured.out
+
+    # Test recursive
+    args = [
+        "list-strategies",
+        "--strategy-path",
+        str(Path(__file__).parent.parent / "strategy" / "strats"),
+        '--no-color',
+        '--recursive-strategy-search'
+    ]
+    pargs = get_args(args)
+    # pargs['config'] = None
+    start_list_strategies(pargs)
+    captured = capsys.readouterr()
+    assert "TestStrategyLegacyV1" in captured.out
+    assert "legacy_strategy_v1.py" in captured.out
+    assert "StrategyTestV2" in captured.out
+    assert "TestStrategyNoImplements" in captured.out
+    assert str(Path("broken_strats/broken_futures_strategies.py")) in captured.out
 
 
 def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys):