From f5d09ed0b072c00e219f03975632e5b77b94ae69 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sat, 25 Sep 2021 20:16:38 +0200 Subject: [PATCH] feat: support loading strategies from packages using the python import logic This is an improvement to simply checking subdirectories for python files as we can use the entirety of python import logic. It's thus possible to import from other files within the `strategies` folder. Additionally, this allows sharing strategies with more data than just python files (e.g a README.md). --- docs/strategy-advanced.md | 47 +++++++ freqtrade/resolvers/iresolver.py | 116 ++++++++++++++++++ .../strats/SampleStrategy/__init__.py | 1 + tests/strategy/strats/deep/__init__.py | 17 +++ .../strategy/strats/deep/package/__init__.py | 1 + tests/strategy/test_strategy_loading.py | 24 ++++ 6 files changed, 206 insertions(+) create mode 120000 tests/strategy/strats/SampleStrategy/__init__.py create mode 100644 tests/strategy/strats/deep/__init__.py create mode 120000 tests/strategy/strats/deep/package/__init__.py diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 4409af6ea..7ce7007fe 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -11,6 +11,53 @@ If you're just getting started, please be familiar with the methods described in !!! Tip You can get a strategy template containing all below methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced` +## Packages + +It's possible to use python packages as well, so this folder structure is valid too: + +``` +user_data/ +└──strategies/ + ├── AwesomeStrategy + │ └── __init__.py + └── various_strategies + ├── __init__.py + ├── advanced + │ ├── advanced_a.py + │ ├── advanced_b.py + │ └── __init__.py + └── simple + └── __init__.py +``` + +A few examples of how to use the strategies therein: + +```shell +# This will import AwesomeStrategy.AwesomeStrategy +freqtrade trade --strategy AwesomeStrategy + +### +# The rest are all converted to their python `import ...` equivalents + +# from various_strategies.simple import SimpleStrategy +freqtrade trade --strategy various_strategies.simple.SimpleStrategy +# from various_strategies.simple import SimpleStrategy2 +freqtrade trade --strategy various_strategies.simple.SimpleStrategy + +# from various_strategies.advanced.advanced_a import AdvancedStrategy +freqtrade trade --strategy various_strategies.advanced.advanced_a.AdvancedStrategy +# from various_strategies.advanced.advanced_b import AdvancedStrategy +freqtrade trade --strategy various_strategies.advanced.advanced_b.AdvancedStrategy +``` + +You can also write common methods and import then within the files you need. + +!!! Warning + Internally this is done by putting `user_data/strategies` as first path + into `sys.path` / `PYTHONPATH`. + Beware how you name your modules! For example `user_data/strategies/math.py` + will otherwise take precedence over the default `math` module and probably make `math.pi` fail. + ## Storing information Storing information can be accomplished by creating a new dictionary within the strategy class. diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 2cccec70a..ba25b39e9 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -6,7 +6,9 @@ This module load custom objects import importlib.util import inspect import logging +import sys from pathlib import Path +from types import ModuleType from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union from freqtrade.exceptions import OperationalException @@ -83,6 +85,33 @@ class IResolver: def _search_object(cls, directory: Path, *, object_name: str, add_source: bool = False ) -> Union[Tuple[Any, Path], Tuple[None, None]]: """ + Import object from the given directory + + ..see:: `Importer.import_item`_ + + :param directory: relative or absolute directory path + :param object_name: ClassName or dot separated import path + :return: object class + """ + logger.debug(f"Attempting import of {cls.object_type.__name__} {object_name} " + f"from '{directory}'") + try: + with Importer(directory) as importer: + item, module = importer.import_item(object_name) + item.__file__ = module.__file__ + module_path = Path(module.__file__) + if add_source: + item.__source__ = module_path.read_text() + return item, module_path + except ImportError: + logger.debug("Falling back to old resolution method") + return cls._old_search_object(directory, object_name=object_name, add_source=add_source) + + @classmethod + def _old_search_object( + cls, directory: Path, *, object_name: str, add_source: bool = False + ) -> Union[Tuple[Any, Path], Tuple[None, None]]: + """ Search for the objectname in the given directory :param directory: relative or absolute directory path :param object_name: ClassName of the object to load @@ -182,3 +211,90 @@ class IResolver: 'location': entry, }) return objects + + +class Importer: + """ + A context manager to help with importing files objects from modules/packages in a given path + + This is necessary as sys.path is modified and the changes shouldn't persist + """ + + def __init__(self, path: Path): + """ + :param path: Where to search from + """ + self.path = path + + def __enter__(self): + """Modifies sys.path in order to import items using the python import logic""" + sys.path.insert(0, str(self.path)) + return self + + def import_item(self, name: str) -> Tuple[type, ModuleType]: + """ + Tries to import an item from a module + + ..Examples:: + + import_item("MyClass") --> from MyClass import MyClass + import_item("path.to.MyClass") --> from path.to import MyClass + + Two approaches will be attempted with two variants + + For a simple name (no dots) the equivalent of `from import `: + + - .py containing `class ` + - /__init.py containing `class ` + + For a dot-separated name the equivalent of + `from import ` + + - //.py containing `class + - //__init.py containing `class ` + + :param name: A valid python module and import path or name + :return: The item to import + :throws: ImportError + """ + + module = self.import_module(name) + try: + item = getattr(module, name.rpartition(".")[2]) + except AttributeError: + raise ImportError(name=name, path=str(self.path)) + + return item, module + + @staticmethod + def import_module(name: str) -> ModuleType: + """ + Interprets the given string as a path to import a module + + :param: name: the simple name of a module, or a path to a module or class within a module + "some_module" --> import some_module + "path.to.some_module.SomeClass" --> from path.to.some_module import Someclass + :returns: A module or throws an ImportError. + ..see: https://docs.python.org/3/library/functions.html#__import__ + """ + import_path, _, from_item = name.rpartition(".") + kwargs: Dict[str, Any] = {} + if import_path: + kwargs["name"] = import_path + kwargs["fromlist"] = [from_item] + else: + kwargs["name"] = from_item + kwargs.update( + { + "globals": globals(), + "locals": locals(), + } + ) + # We need to reload here as we are dynamically importing and want to ignore the module cache + return importlib.reload(importlib.__import__(**kwargs)) + + def __exit__(self, exc_type, exc_val, exc_tb): + """Undo up change to sys.path after all is said and done""" + str_path = str(self.path) + if str_path in sys.path: + sys.path.remove(str_path) diff --git a/tests/strategy/strats/SampleStrategy/__init__.py b/tests/strategy/strats/SampleStrategy/__init__.py new file mode 120000 index 000000000..d11f080ff --- /dev/null +++ b/tests/strategy/strats/SampleStrategy/__init__.py @@ -0,0 +1 @@ +../../../../freqtrade/templates/sample_strategy.py \ No newline at end of file diff --git a/tests/strategy/strats/deep/__init__.py b/tests/strategy/strats/deep/__init__.py new file mode 100644 index 000000000..d402f6143 --- /dev/null +++ b/tests/strategy/strats/deep/__init__.py @@ -0,0 +1,17 @@ +""" +freqtrade +Copyright (C) 2021 LoveIsGrief + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" diff --git a/tests/strategy/strats/deep/package/__init__.py b/tests/strategy/strats/deep/package/__init__.py new file mode 120000 index 000000000..0e25530e3 --- /dev/null +++ b/tests/strategy/strats/deep/package/__init__.py @@ -0,0 +1 @@ +../../../../../freqtrade/templates/sample_strategy.py \ No newline at end of file diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 2cbc9d0c6..f6f2e5fef 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging +import sys import warnings from base64 import urlsafe_b64encode from pathlib import Path @@ -31,6 +32,29 @@ def test_search_strategy(): assert s is None +def test_search_package_strategy(): + default_location = Path(__file__).parent / 'strats' + + s, _ = StrategyResolver._search_object( + directory=default_location, + object_name='SampleStrategy', + add_source=True, + ) + assert issubclass(s, IStrategy) + + +def test_new_search_strategy(): + default_location = Path(__file__).parent / 'strats' + old_sys_path = sys.path.copy() + s, _ = StrategyResolver._search_object( + directory=default_location, + object_name='deep.package.SampleStrategy', + add_source=True, + ) + assert issubclass(s, IStrategy) + assert old_sys_path == sys.path + + def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False)