diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c9ec30056..6b6c2a772 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -40,7 +40,8 @@ pip install -r requirements-hyperopt.txt ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] - [--recursive-strategy-search] [-i TIMEFRAME] + [--recursive-strategy-search] [--freqaimodel NAME] + [--freqaimodel-path PATH] [-i TIMEFRAME] [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] @@ -53,7 +54,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] [--hyperopt-loss NAME] [--disable-param-export] - [--ignore-missing-spaces] + [--ignore-missing-spaces] [--analyze-per-epoch] optional arguments: -h, --help show this help message and exit @@ -129,6 +130,7 @@ optional arguments: --ignore-missing-spaces, --ignore-unparameterized-spaces Suppress errors for any requested Hyperopt spaces that do not contain any parameters. + --analyze-per-epoch Run populate_indicators once per epoch. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -154,6 +156,10 @@ Strategy arguments: --recursive-strategy-search Recursively search for a strategy in the strategies folder. + --freqaimodel NAME Specify a custom freqaimodels. + --freqaimodel-path PATH + Specify additional lookup path for freqaimodels. + ``` ### Hyperopt checklist @@ -185,7 +191,7 @@ Rarely you may also need to create a [nested class](advanced-hyperopt.md#overrid ### Hyperopt execution logic -Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators. +Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators, unless `--analyze-per-epoch` is specified. Hyperopt will then spawn into different processes (number of processors, or `-j `), and run backtesting over and over again, changing the parameters that are part of the `--spaces` defined. @@ -426,9 +432,10 @@ While this strategy is most likely too simple to provide consistent profit, it s `range` property may also be used with `DecimalParameter` and `CategoricalParameter`. `RealParameter` does not provide this property due to infinite search space. ??? Hint "Performance tip" - By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter. - While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values). - You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space. + During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value. This will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues. + + In either case, you should try to use space ranges as small as possible this will improve CPU/RAM usage in both scenarios. + ## Optimizing protections @@ -879,6 +886,7 @@ To combat these, you have multiple options: * Avoid using `--timeframe-detail` (this loads a lot of additional data into memory). * Reduce the number of parallel processes (`-j `). * Increase the memory of your machine. +* Use `--analyze-per-epoch` if you're using a lot of parameters with `.range` functionality. ## The objective has been evaluated at this point before. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 266095cfa..37ce17f21 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -34,7 +34,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_loss", "disableparamexport", - "hyperopt_ignore_missing_space"] + "hyperopt_ignore_missing_space", "analyze_per_epoch"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index b9e9e26f7..3d094da36 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -255,6 +255,13 @@ AVAILABLE_CLI_OPTIONS = { nargs='+', default='default', ), + "analyze_per_epoch": Arg( + '--analyze-per-epoch', + help='Run populate_indicators once per epoch.', + action='store_true', + default=False, + ), + "print_all": Arg( '--print-all', help='Print all results, not only the best ones.', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 0db585acc..7c68ac46c 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -302,6 +302,9 @@ class Configuration: self._args_to_config(config, argname='spaces', logstring='Parameter -s/--spaces detected: {}') + self._args_to_config(config, argname='analyze_per_epoch', + logstring='Parameter --analyze-per-epoch detected.') + self._args_to_config(config, argname='print_all', logstring='Parameter --print-all detected ...') diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index e50ebc4a4..d2f5474fc 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -3,6 +3,7 @@ from freqtrade.enums.backteststate import BacktestState from freqtrade.enums.candletype import CandleType from freqtrade.enums.exitchecktuple import ExitCheckTuple from freqtrade.enums.exittype import ExitType +from freqtrade.enums.hyperoptstate import HyperoptState from freqtrade.enums.marginmode import MarginMode from freqtrade.enums.ordertypevalue import OrderTypeValues from freqtrade.enums.rpcmessagetype import RPCMessageType diff --git a/freqtrade/enums/hyperoptstate.py b/freqtrade/enums/hyperoptstate.py new file mode 100644 index 000000000..6716e123a --- /dev/null +++ b/freqtrade/enums/hyperoptstate.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class HyperoptState(Enum): + """ Hyperopt states """ + STARTUP = 1 + DATALOAD = 2 + INDICATORS = 3 + OPTIMIZE = 4 + + def __str__(self): + return f"{self.name.lower()}" diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cbcf39131..fea2a672f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -24,13 +24,15 @@ from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange +from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss -from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer +from freqtrade.optimize.hyperopt_tools import (HyperoptStateContainer, HyperoptTools, + hyperopt_serializer) from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver @@ -74,10 +76,14 @@ class Hyperopt: self.dimensions: List[Dimension] = [] self.config = config + self.min_date: datetime + self.max_date: datetime self.backtesting = Backtesting(self.config) self.pairlist = self.backtesting.pairlists.whitelist self.custom_hyperopt: HyperOptAuto + self.analyze_per_epoch = self.config.get('analyze_per_epoch', False) + HyperoptStateContainer.set_state(HyperoptState.STARTUP) if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) @@ -290,6 +296,7 @@ class Hyperopt: Called once per epoch to optimize whatever is configured. Keep this function as optimized as possible! """ + HyperoptStateContainer.set_state(HyperoptState.OPTIMIZE) backtest_start_time = datetime.now(timezone.utc) params_dict = self._get_params_dict(self.dimensions, raw_params) @@ -321,6 +328,10 @@ class Hyperopt: with self.data_pickle_file.open('rb') as f: processed = load(f, mmap_mode='r') + if self.analyze_per_epoch: + # Data is not yet analyzed, rerun populate_indicators. + processed = self.advise_and_trim(processed) + bt_results = self.backtesting.backtest( processed=processed, start_date=self.min_date, @@ -406,22 +417,33 @@ class Hyperopt: def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) - def prepare_hyperopt_data(self) -> None: - data, timerange = self.backtesting.load_bt_data() - self.backtesting.load_bt_data_detail() - logger.info("Dataload complete. Calculating indicators") - + def advise_and_trim(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: preprocessed = self.backtesting.strategy.advise_all_indicators(data) # Trim startup period from analyzed dataframe to get correct dates for output. - processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup) + processed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup) self.min_date, self.max_date = get_timerange(processed) + return processed - logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(self.max_date - self.min_date).days} days)..') - # Store non-trimmed data - will be trimmed after signal generation. - dump(preprocessed, self.data_pickle_file) + def prepare_hyperopt_data(self) -> None: + HyperoptStateContainer.set_state(HyperoptState.DATALOAD) + data, self.timerange = self.backtesting.load_bt_data() + self.backtesting.load_bt_data_detail() + logger.info("Dataload complete. Calculating indicators") + + if not self.analyze_per_epoch: + HyperoptStateContainer.set_state(HyperoptState.INDICATORS) + + preprocessed = self.advise_and_trim(data) + + logger.info(f'Hyperopting with data from ' + f'{self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(self.max_date - self.min_date).days} days)..') + # Store non-trimmed data - will be trimmed after signal generation. + dump(preprocessed, self.data_pickle_file) + else: + dump(data, self.data_pickle_file) def get_asked_points(self, n_points: int) -> Tuple[List[List[Any]], List[bool]]: """ diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index ab6ef013b..9b022d519 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -13,6 +13,7 @@ from colorama import Fore, Style from pandas import isna, json_normalize from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES +from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs @@ -32,6 +33,15 @@ def hyperopt_serializer(x): return str(x) +class HyperoptStateContainer(): + """ Singleton class to track state of hyperopt""" + state: HyperoptState = HyperoptState.OPTIMIZE + + @classmethod + def set_state(cls, value: HyperoptState): + cls.state = value + + class HyperoptTools(): @staticmethod diff --git a/freqtrade/strategy/parameters.py b/freqtrade/strategy/parameters.py index 83dd41de9..c6037ae0b 100644 --- a/freqtrade/strategy/parameters.py +++ b/freqtrade/strategy/parameters.py @@ -7,6 +7,9 @@ from abc import ABC, abstractmethod from contextlib import suppress from typing import Any, Optional, Sequence, Union +from freqtrade.enums.hyperoptstate import HyperoptState +from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer + with suppress(ImportError): from skopt.space import Integer, Real, Categorical @@ -57,6 +60,13 @@ class BaseParameter(ABC): Get-space - will be used by Hyperopt to get the hyperopt Space """ + def can_optimize(self): + return ( + self.in_space + and self.optimize + and HyperoptStateContainer.state != HyperoptState.OPTIMIZE + ) + class NumericParameter(BaseParameter): """ Internal parameter used for Numeric purposes """ @@ -133,7 +143,7 @@ class IntParameter(NumericParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): # Scikit-optimize ranges are "inclusive", while python's "range" is exclusive return range(self.low, self.high + 1) else: @@ -212,7 +222,7 @@ class DecimalParameter(NumericParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): low = int(self.low * pow(10, self._decimals)) high = int(self.high * pow(10, self._decimals)) + 1 return [round(n * pow(0.1, self._decimals), self._decimals) for n in range(low, high)] @@ -261,7 +271,7 @@ class CategoricalParameter(BaseParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): return self.opt_range else: return [self.value] diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 83f7d19b7..65ee05d71 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -12,7 +12,9 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data from freqtrade.enums import ExitCheckTuple, ExitType, SignalDirection +from freqtrade.enums.hyperoptstate import HyperoptState from freqtrade.exceptions import OperationalException, StrategyError +from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer from freqtrade.optimize.space import SKDecimal from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver @@ -859,7 +861,9 @@ def test_strategy_safe_wrapper_trade_copy(fee): def test_hyperopt_parameters(): + HyperoptStateContainer.set_state(HyperoptState.INDICATORS) from skopt.space import Categorical, Integer, Real + with pytest.raises(OperationalException, match=r"Name is determined.*"): IntParameter(low=0, high=5, default=1, name='hello') @@ -937,6 +941,12 @@ def test_hyperopt_parameters(): assert list(boolpar.range) == [True, False] + HyperoptStateContainer.set_state(HyperoptState.OPTIMIZE) + assert len(list(intpar.range)) == 1 + assert len(list(fltpar.range)) == 1 + assert len(list(catpar.range)) == 1 + assert len(list(boolpar.range)) == 1 + def test_auto_hyperopt_interface(default_conf): default_conf.update({'strategy': 'HyperoptableStrategyV2'})