From 477515c4b5904b81cd9d6a6c99aa93f59daf39a8 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sun, 1 Apr 2018 21:06:50 +1000 Subject: [PATCH] Now using resolver for custom hyperopts --- freqtrade/arguments.py | 6 +- freqtrade/constants.py | 2 +- freqtrade/optimize/custom_hyperopt.py | 153 -------------------------- freqtrade/optimize/hyperopt.py | 7 +- freqtrade/optimize/resolver.py | 104 +++++++++++++++++ user_data/hyperopts/test_hyperopt.py | 4 - 6 files changed, 112 insertions(+), 164 deletions(-) delete mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/resolver.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a36b3e66a..429a7ca5a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -105,12 +105,12 @@ class Arguments(object): metavar='PATH', ) self.parser.add_argument( - '--hyperopt', - help='specify hyperopt file (default: %(default)s)', + '--customhyperopt', + help='specify hyperopt class name (default: %(default)s)', dest='hyperopt', default=Constants.DEFAULT_HYPEROPT, type=str, - metavar='PATH', + metavar='NAME', ) self.parser.add_argument( '--dynamic-whitelist', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a03efdc4b..e76c54ba2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,7 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' -DEFAULT_HYPEROPT = 'default_hyperopt' +DEFAULT_HYPEROPT = 'DefaultHyperOpts' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py deleted file mode 100644 index 80168c46e..000000000 --- a/freqtrade/optimize/custom_hyperopt.py +++ /dev/null @@ -1,153 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom hyperopts -""" -import importlib -import logging -import os -import sys -from typing import Dict, Any, Callable - -from pandas import DataFrame - -from freqtrade.constants import Constants -from freqtrade.optimize.interface import IHyperOpt - -sys.path.insert(0, r'../../user_data/hyperopts') - -logger = logging.getLogger(__name__) - - -class CustomHyperOpt(object): - """ - This class contains all the logic to load custom hyperopt class - """ - def __init__(self, config: dict = {}) -> None: - """ - Load the custom class from config parameter - :param config: - :return: - """ - - # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - if 'hyperopt' in config: - hyperopt = config['hyperopt'] - else: - hyperopt = Constants.DEFAULT_HYPEROPT - - # Load the hyperopt - self._load_hyperopt(hyperopt) - - def _load_hyperopt(self, hyperopt_name: str) -> None: - """ - Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt - Set the object into self.custom_hyperopt - :param hyperopt_name: name of the module to import - :return: None - """ - - try: - # Start by sanitizing the file name (remove any extensions) - hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) - - # Search where can be the hyperopt file - path = self._search_hyperopt(filename=hyperopt_name) - - # Load the hyperopt - self.custom_hyperopt = self._load_class(path + hyperopt_name) - - # Fallback to the default hyperopt - except (ImportError, TypeError) as error: - logger.error( - "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" - " or contains Python code errors", - hyperopt_name - ) - logger.error( - "The error is:\n%s.", - error - ) - - def _load_class(self, filename: str) -> IHyperOpt: - """ - Import a hyperopt as a module - :param filename: path to the hyperopt (path from freqtrade/optimize/) - :return: return the hyperopt class - """ - module = importlib.import_module(filename, __package__) - custom_hyperopt = getattr(module, module.class_name) - - logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) - return custom_hyperopt() - - @staticmethod - def _sanitize_module_name(filename: str) -> str: - """ - Remove any extension from filename - :param filename: filename to sanatize - :return: return the filename without extensions - """ - filename = os.path.basename(filename) - filename = os.path.splitext(filename)[0] - return filename - - @staticmethod - def _search_hyperopt(filename: str) -> str: - """ - Search for the hyperopt file in different folder - 1. search into the user_data/hyperopts folder - 2. search into the freqtrade/optimize folder - 3. if nothing found, return None - :param hyperopt_name: module name to search - :return: module path where is the hyperopt - """ - pwd = os.path.dirname(os.path.realpath(__file__)) + '/' - user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') - hyperopt_folder = os.path.join(pwd, filename + '.py') - - path = None - if os.path.isfile(user_data): - path = 'user_data.hyperopts.' - elif os.path.isfile(hyperopt_folder): - path = '.' - - return path - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell hyperopt - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :return: a Dataframe with all mandatory indicators for the strategies - """ - return self.custom_hyperopt.populate_indicators(dataframe) - - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a buy strategy generator - """ - return self.custom_hyperopt.buy_strategy_generator(params) - - def indicator_space(self) -> Dict[str, Any]: - """ - Create an indicator space - """ - return self.custom_hyperopt.indicator_space() - - def generate_roi_table(self, params: Dict) -> Dict[int, float]: - """ - Create an roi table - """ - return self.custom_hyperopt.generate_roi_table(params) - - def stoploss_space(self) -> Dict[str, Any]: - """ - Create a stoploss space - """ - return self.custom_hyperopt.stoploss_space() - - def roi_space(self) -> Dict[str, Any]: - """ - Create a roi space - """ - return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5e23458d7..07c24ff18 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,8 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.custom_hyperopt import CustomHyperOpt +from freqtrade.optimize.resolver import HyperOptResolver + logger = logging.getLogger(__name__) @@ -40,8 +41,8 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - - self.custom_hyperopt = CustomHyperOpt(self.config) + self.config = config + self.custom_hyperopt = HyperOptResolver(self.config).hyperopt # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/resolver.py new file mode 100644 index 000000000..78637483b --- /dev/null +++ b/freqtrade/optimize/resolver.py @@ -0,0 +1,104 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib.util +import inspect +import logging +import os +from typing import Optional, Dict, Type + +from freqtrade.constants import Constants +from freqtrade.optimize.interface import IHyperOpt + + +logger = logging.getLogger(__name__) + + +class HyperOptResolver(object): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['hyperopt'] + + def __init__(self, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + hyperopt_name = config.get('hyperopt') or Constants.DEFAULT_HYPEROPT + self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + + def _load_hyperopt( + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> Optional[IHyperOpt]: + """ + Search and loads the specified hyperopt. + :param hyperopt_name: name of the module to import + :param extra_dir: additional directory to search for the given hyperopt + :return: HyperOpt instance or None + """ + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'hyperopts'), + current_path, + ] + + if extra_dir: + # Add extra hyperopt directory on top of search paths + abs_paths.insert(0, extra_dir) + + for path in abs_paths: + hyperopt = self._search_hyperopt(path, hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, path) + return hyperopt + + raise ImportError( + "Impossible to load Hyperopt '{}'. This class does not exist" + " or contains Python code errors".format(hyperopt_name) + ) + + @staticmethod + def _get_valid_hyperopts(module_path: str, hyperopt_name: str) -> Optional[Type[IHyperOpt]]: + """ + Returns a list of all possible hyperopts for the given module_path + :param module_path: absolute path to the module + :param hyperopt_name: Class name of the hyperopt + :return: Tuple with (name, class) or None + """ + + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + valid_hyperopts_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if hyperopt_name == name and IHyperOpt in obj.__bases__ + ) + return next(valid_hyperopts_gen, None) + + @staticmethod + def _search_hyperopt(directory: str, hyperopt_name: str) -> Optional[IHyperOpt]: + """ + Search for the hyperopt_name in the given directory + :param directory: relative or absolute directory path + :return: name of the hyperopt class + """ + logger.debug('Searching for hyperopt %s in \'%s\'', hyperopt_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + hyperopt = HyperOptResolver._get_valid_hyperopts( + os.path.abspath(os.path.join(directory, entry)), hyperopt_name + ) + if hyperopt: + return hyperopt() + return None diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py index 0ee43c64f..b742af6c9 100644 --- a/user_data/hyperopts/test_hyperopt.py +++ b/user_data/hyperopts/test_hyperopt.py @@ -15,10 +15,6 @@ from freqtrade.indicator_helpers import fishers_inverse from freqtrade.optimize.interface import IHyperOpt -# Update this variable if you change the class name -class_name = 'TestHyperOpt' - - # This class is a sample. Feel free to customize it. class TestHyperOpt(IHyperOpt): """