From b736691e0e5a80a54108f20e3d438d265a281ca8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 16:18:28 +0200 Subject: [PATCH 01/12] Remove hyperopt --continue --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 7 ------- freqtrade/configuration/configuration.py | 3 --- freqtrade/optimize/hyperopt.py | 5 +---- tests/optimize/test_hyperopt.py | 11 ----------- 5 files changed, 2 insertions(+), 26 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index b61a4933e..a9ad6bc63 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -26,7 +26,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "use_max_market_positions", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", - "hyperopt_continue", "hyperopt_loss"] + "hyperopt_loss"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 81b8de1af..eeadd62f6 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -252,13 +252,6 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=1, ), - "hyperopt_continue": Arg( - "--continue", - help="Continue hyperopt from previous runs. " - "By default, temporary files will be removed and hyperopt will start from scratch.", - default=False, - action='store_true', - ), "hyperopt_loss": Arg( '--hyperopt-loss', help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 930917fae..4a53ef7e5 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -295,9 +295,6 @@ class Configuration: self._args_to_config(config, argname='hyperopt_min_trades', logstring='Parameter --min-trades detected: {}') - self._args_to_config(config, argname='hyperopt_continue', - logstring='Hyperopt continue: {}') - self._args_to_config(config, argname='hyperopt_loss', logstring='Using Hyperopt loss class name: {}') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 37de3bc4b..8682aa08d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -86,10 +86,7 @@ class Hyperopt: self.current_best_loss = 100 - if not self.config.get('hyperopt_continue'): - self.clean_hyperopt() - else: - logger.info("Continuing on previous hyperopt results.") + self.clean_hyperopt() self.num_epochs_saved = 0 diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index d58b91209..a35b8b655 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -839,17 +839,6 @@ def test_clean_hyperopt(mocker, hyperopt_conf, caplog): assert log_has(f"Removing `{h.data_pickle_file}`.", caplog) -def test_continue_hyperopt(mocker, hyperopt_conf, caplog): - patch_exchange(mocker) - hyperopt_conf.update({'hyperopt_continue': True}) - mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True)) - unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock()) - Hyperopt(hyperopt_conf) - - assert unlinkmock.call_count == 0 - assert log_has("Continuing on previous hyperopt results.", caplog) - - def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', From 7a652b07d587acfef3cc5cd090289a38948ea475 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 16:21:55 +0200 Subject: [PATCH 02/12] UPdate documentation to remove --continue --- docs/bot-usage.md | 5 +---- docs/hyperopt.md | 15 ++++++--------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 4a4496bbc..62c515b44 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -303,7 +303,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] [--dmmp] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] - [--continue] [--hyperopt-loss NAME] + [--hyperopt-loss NAME] optional arguments: -h, --help show this help message and exit @@ -349,9 +349,6 @@ optional arguments: reproducible hyperopt results. --min-trades INT Set minimal desired number of trades for evaluations in the hyperopt optimization path (default: 1). - --continue Continue hyperopt from previous runs. By default, - temporary files will be removed and hyperopt will - start from scratch. --hyperopt-loss NAME Specify the class name of the hyperopt loss function class (IHyperOptLoss). Different functions can generate completely different results, since the diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 3f7a27ef0..6162f766a 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -102,7 +102,7 @@ There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower Bollinger band". -Hyperoptimization will, for each eval round, pick one trigger and possibly +Hyper-optimization will, for each epoch round, pick one trigger and possibly multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower Bollinger band, BUT only if ADX > 10*". @@ -240,10 +240,7 @@ running at least several thousand evaluations. The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. !!! Note - By default, hyperopt will erase previous results and start from scratch. Continuation can be archived by using `--continue`. - -!!! Warning - When switching parameters or changing configuration options, make sure to not use the argument `--continue` so temporary results can be removed. + By default, hyperopt will erase previous results and start from scratch. ### Execute Hyperopt with different historical data source @@ -253,7 +250,7 @@ uses data from directory `user_data/data`. ### Running Hyperopt with Smaller Testset -Use the `--timerange` argument to change how much of the testset you want to use. +Use the `--timerange` argument to change how much of the test-set you want to use. For example, to use one month of data, pass the following parameter to the hyperopt call: ```bash @@ -318,7 +315,7 @@ The initial state for generation of these random values (random state) is contro If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used. -If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyperoptimization results with same random state value used. +If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used. ## Understand the Hyperopt Result @@ -371,7 +368,7 @@ By default, hyperopt prints colorized results -- epochs with positive profit are You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option. !!! Note "Windows and color output" - Windows does not support color-output nativly, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL. + Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL. ### Understand Hyperopt ROI results @@ -494,7 +491,7 @@ Override the `trailing_space()` method and define the desired range in it if you ## Show details of Hyperopt results -After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` subcommands. The usage of these subcommands is described in the [Utils](utils.md#list-hyperopt-results) chapter. +After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` sub-commands. The usage of these sub-commands is described in the [Utils](utils.md#list-hyperopt-results) chapter. ## Validate backtesting results From ff96cf154c5c8b9c9595f695b0f048765af59be8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 16:33:26 +0200 Subject: [PATCH 03/12] Keep hyperopt result history --- freqtrade/optimize/hyperopt.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8682aa08d..c07fa1788 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,6 +10,7 @@ import logging import random import warnings from collections import OrderedDict +from datetime import datetime from math import ceil from operator import itemgetter from pathlib import Path @@ -25,16 +26,15 @@ from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) from pandas import DataFrame, isna, json_normalize -from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframe from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException -from freqtrade.misc import plural, round_dict +from freqtrade.misc import file_dump_json, plural, round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 -from freqtrade.optimize.hyperopt_loss_interface import \ - IHyperOptLoss # noqa: F401 +from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, HyperOptResolver) from freqtrade.strategy import IStrategy @@ -77,9 +77,9 @@ class Hyperopt: self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function - + time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") self.results_file = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + 'hyperopt_results' / f'hyperopt_results_{time_now}.pickle') self.data_pickle_file = (self.config['user_data_dir'] / 'hyperopt_results' / 'hyperopt_tickerdata.pkl') self.total_epochs = config.get('epochs', 0) @@ -162,6 +162,9 @@ class Hyperopt: self.num_epochs_saved = num_epochs logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " f"saved to '{self.results_file}'.") + # Store hyperopt filename + latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN) + file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)}) @staticmethod def _read_results(results_file: Path) -> List: From c42a924df85150092d106bfd739f78a325513f25 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 16:50:22 +0200 Subject: [PATCH 04/12] Load latest file --- freqtrade/commands/hyperopt_commands.py | 8 ++-- freqtrade/data/btanalysis.py | 50 +++++++++++++++++++++++-- freqtrade/optimize/hyperopt.py | 2 - tests/optimize/test_hyperopt.py | 2 + 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 4fae51e28..de8764369 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -7,6 +7,7 @@ from colorama import init as colorama_init from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode +from freqtrade.data.btanalysis import get_latest_hyperopt_file logger = logging.getLogger(__name__) @@ -40,8 +41,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: 'filter_max_objective': config.get('hyperopt_list_max_objective', None), } - results_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + results_file = get_latest_hyperopt_file(config['user_data_dir'] / 'hyperopt_results') # Previous evaluations epochs = Hyperopt.load_previous_results(results_file) @@ -80,8 +80,8 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: print_json = config.get('print_json', False) no_header = config.get('hyperopt_show_no_header', False) - results_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + results_file = get_latest_hyperopt_file(config['user_data_dir'] / 'hyperopt_results') + n = config.get('hyperopt_show_index', -1) filteroptions = { diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 2d45a7222..55e4f11c4 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -21,10 +21,11 @@ BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "open_rate", "close_rate", "open_at_end", "sell_reason"] -def get_latest_backtest_filename(directory: Union[Path, str]) -> str: +def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str: """ Get latest backtest export based on '.last_result.json'. :param directory: Directory to search for last result + :param variant: 'backtest' or 'hyperopt' - the method to return :return: string containing the filename of the latest backtest result :raises: ValueError in the following cases: * Directory does not exist @@ -44,10 +45,53 @@ def get_latest_backtest_filename(directory: Union[Path, str]) -> str: with filename.open() as file: data = json_load(file) - if 'latest_backtest' not in data: + if f'latest_{variant}' not in data: raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.") - return data['latest_backtest'] + return data[f'latest_{variant}'] + + +def get_latest_backtest_filename(directory: Union[Path, str]) -> str: + """ + Get latest backtest export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest backtest result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + return get_latest_optimize_filename(directory, 'backtest') + + +def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str: + """ + Get latest hyperopt export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest hyperopt result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + try: + return get_latest_optimize_filename(directory, 'hyperopt') + except ValueError: + # Return default (legacy) pickle filename + return 'hyperopt_results.pickle' + + +def get_latest_hyperopt_file(directory: Union[Path, str]) -> Path: + """ + Get latest hyperopt export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest hyperopt result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + return directory / get_latest_hyperopt_filename(directory) def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c07fa1788..9d16bc6ba 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -657,8 +657,6 @@ class Hyperopt: self.backtesting.strategy.dp = None # type: ignore IStrategy.dp = None # type: ignore - self.epochs = self.load_previous_results(self.results_file) - cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index a35b8b655..ec911c113 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -497,6 +497,7 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> None: epochs = create_results(mocker, hyperopt, testdatadir) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) + mock_dump_json = mocker.patch('freqtrade.optimize.hyperopt.file_dump_json', return_value=None) results_file = testdatadir / 'optimize' / 'ut_results.pickle' caplog.set_level(logging.DEBUG) @@ -505,6 +506,7 @@ def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> Non hyperopt._save_results() assert log_has(f"1 epoch saved to '{results_file}'.", caplog) mock_dump.assert_called_once() + mock_dump_json.assert_called_once() hyperopt.epochs = epochs + epochs hyperopt._save_results() From 3cb1a9a5a959a0c34ccf50219c5cb31aa194a557 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 17:00:23 +0200 Subject: [PATCH 05/12] Support loading results from a specific hyperopt history file --- freqtrade/commands/arguments.py | 4 ++-- freqtrade/commands/cli_options.py | 6 ++++++ freqtrade/commands/hyperopt_commands.py | 9 +++++++-- freqtrade/configuration/configuration.py | 3 +++ freqtrade/data/btanalysis.py | 5 ++++- tests/data/test_btanalysis.py | 9 +++++++++ 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index a9ad6bc63..72f60eb32 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -75,10 +75,10 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", "hyperopt_list_min_objective", "hyperopt_list_max_objective", "print_colorized", "print_json", "hyperopt_list_no_details", - "export_csv"] + "hyperoptexportfilename", "export_csv"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", - "print_json", "hyperopt_show_no_header"] + "print_json", "hyperoptexportfilename", "hyperopt_show_no_header"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index eeadd62f6..cd0bc426d 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -263,6 +263,12 @@ AVAILABLE_CLI_OPTIONS = { metavar='NAME', default=constants.DEFAULT_HYPEROPT_LOSS, ), + "hyperoptexportfilename": Arg( + '--hyperopt-filename', + help='Hyperopt result filename.' + 'Example: `--hyperopt-filename=hyperopt_results_2020-09-27_16-20-48.pickle`', + metavar='PATH', + ), # List exchanges "print_one_column": Arg( '-1', '--one-column', diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index de8764369..9075d19d2 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -41,7 +41,9 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: 'filter_max_objective': config.get('hyperopt_list_max_objective', None), } - results_file = get_latest_hyperopt_file(config['user_data_dir'] / 'hyperopt_results') + results_file = get_latest_hyperopt_file( + config['user_data_dir'] / 'hyperopt_results', + config.get('hyperoptexportfilename')) # Previous evaluations epochs = Hyperopt.load_previous_results(results_file) @@ -80,7 +82,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: print_json = config.get('print_json', False) no_header = config.get('hyperopt_show_no_header', False) - results_file = get_latest_hyperopt_file(config['user_data_dir'] / 'hyperopt_results') + results_file = get_latest_hyperopt_file( + config['user_data_dir'] / 'hyperopt_results', + config.get('hyperoptexportfilename')) + n = config.get('hyperopt_show_index', -1) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 4a53ef7e5..d022c1e62 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -263,6 +263,9 @@ class Configuration: self._args_to_config(config, argname='hyperopt_path', logstring='Using additional Hyperopt lookup path: {}') + self._args_to_config(config, argname='hyperoptexportfilename', + logstring='Using hyperopt file: {}') + self._args_to_config(config, argname='epochs', logstring='Parameter --epochs detected ... ' 'Will run Hyperopt with for {} epochs ...' diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 55e4f11c4..d4edd41a8 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -81,7 +81,7 @@ def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str: return 'hyperopt_results.pickle' -def get_latest_hyperopt_file(directory: Union[Path, str]) -> Path: +def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str = None) -> Path: """ Get latest hyperopt export based on '.last_result.json'. :param directory: Directory to search for last result @@ -91,6 +91,9 @@ def get_latest_hyperopt_file(directory: Union[Path, str]) -> Path: * `directory/.last_result.json` does not exist * `directory/.last_result.json` has the wrong content """ + + if predef_filename: + return directory / predef_filename return directory / get_latest_hyperopt_filename(directory) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 564dae0b1..43b1a7a4b 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -15,6 +15,7 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, create_cum_profit, extract_trades_of_period, get_latest_backtest_filename, + get_latest_hyperopt_file, load_backtest_data, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history @@ -43,6 +44,14 @@ def test_get_latest_backtest_filename(testdatadir, mocker): get_latest_backtest_filename(testdatadir) +def test_get_latest_hyperopt_file(testdatadir, mocker): + res = get_latest_hyperopt_file(testdatadir / 'does_not_exist', 'testfile.pickle') + assert res == testdatadir / 'does_not_exist/testfile.pickle' + + res = get_latest_hyperopt_file(testdatadir.parent) + assert res == testdatadir.parent / "hyperopt_results.pickle" + + def test_load_backtest_data_old_format(testdatadir): filename = testdatadir / "backtest-result_test.json" From 8de9c46110f6420d12570d55a07b6c6d8c3bc85a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 17:09:33 +0200 Subject: [PATCH 06/12] Document hyperopt-filename usage --- docs/hyperopt.md | 3 +- docs/utils.md | 62 +++++++++++++++++++++++-------- freqtrade/commands/cli_options.py | 2 +- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 6162f766a..8bfd2cf7d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -240,7 +240,8 @@ running at least several thousand evaluations. The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. !!! Note - By default, hyperopt will erase previous results and start from scratch. + Hyperopt will store hyperopt results with the timestamp of the hyperopt start time. + Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename ` to read and display older hyperopt results. ### Execute Hyperopt with different historical data source diff --git a/docs/utils.md b/docs/utils.md index 8c7e381ff..409bcc134 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -423,7 +423,7 @@ freqtrade test-pairlist --config config.json --quote USDT BTC ## List Hyperopt results -You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` subcommand. +You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` sub-command. ``` usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] @@ -432,10 +432,11 @@ usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--max-trades INT] [--min-avg-time FLOAT] [--max-avg-time FLOAT] [--min-avg-profit FLOAT] [--max-avg-profit FLOAT] - [--min-total-profit FLOAT] [--max-total-profit FLOAT] + [--min-total-profit FLOAT] + [--max-total-profit FLOAT] [--min-objective FLOAT] [--max-objective FLOAT] [--no-color] [--print-json] [--no-details] - [--export-csv FILE] + [--hyperopt-filename PATH] [--export-csv FILE] optional arguments: -h, --help show this help message and exit @@ -443,24 +444,27 @@ optional arguments: --profitable Select only profitable epochs. --min-trades INT Select epochs with more than INT trades. --max-trades INT Select epochs with less than INT trades. - --min-avg-time FLOAT Select epochs on above average time. - --max-avg-time FLOAT Select epochs on under average time. + --min-avg-time FLOAT Select epochs above average time. + --max-avg-time FLOAT Select epochs below average time. --min-avg-profit FLOAT - Select epochs on above average profit. + Select epochs above average profit. --max-avg-profit FLOAT - Select epochs on below average profit. + Select epochs below average profit. --min-total-profit FLOAT - Select epochs on above total profit. + Select epochs above total profit. --max-total-profit FLOAT - Select epochs on below total profit. + Select epochs below total profit. --min-objective FLOAT - Select epochs on above objective (- is added by default). + Select epochs above objective. --max-objective FLOAT - Select epochs on below objective (- is added by default). + Select epochs below objective. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. - --print-json Print best result detailization in JSON format. + --print-json Print output in JSON format. --no-details Do not print best epoch details. + --hyperopt-filename FILENAME + Hyperopt result filename.Example: `--hyperopt- + filename=hyperopt_results_2020-09-27_16-20-48.pickle` --export-csv FILE Export to CSV-File. This will disable table print. Example: --export-csv hyperopt.csv @@ -480,7 +484,11 @@ Common arguments: --userdir PATH, --user-data-dir PATH Path to userdata directory. ``` - + +!!! Note + `hyperopt-list` will automatically use the latest available hyperopt results file. + You can override this using the `--hyperopt-filename` argument, and specify another, available filename (without path!). + ### Examples List all results, print details of the best result at the end: @@ -501,17 +509,41 @@ You can show the details of any hyperoptimization epoch previously evaluated by usage: freqtrade hyperopt-show [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--best] [--profitable] [-n INT] [--print-json] - [--no-header] + [--hyperopt-filename PATH] [--no-header] optional arguments: -h, --help show this help message and exit --best Select only best epochs. --profitable Select only profitable epochs. -n INT, --index INT Specify the index of the epoch to print details for. - --print-json Print best result detailization in JSON format. + --print-json Print output in JSON format. + --hyperopt-filename FILENAME + Hyperopt result filename.Example: `--hyperopt- + filename=hyperopt_results_2020-09-27_16-20-48.pickle` --no-header Do not print epoch details header. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + ``` +!!! Note + `hyperopt-show` will automatically use the latest available hyperopt results file. + You can override this using the `--hyperopt-filename` argument, and specify another, available filename (without path!). + ### Examples Print details for the epoch 168 (the number of the epoch is shown by the `hyperopt-list` subcommand or by Hyperopt itself during hyperoptimization run): diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index cd0bc426d..3c5775768 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -267,7 +267,7 @@ AVAILABLE_CLI_OPTIONS = { '--hyperopt-filename', help='Hyperopt result filename.' 'Example: `--hyperopt-filename=hyperopt_results_2020-09-27_16-20-48.pickle`', - metavar='PATH', + metavar='FILENAME', ), # List exchanges "print_one_column": Arg( From 5769b9244f83343c66309e7b4c4eb73cce03f1b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 19:34:47 +0200 Subject: [PATCH 07/12] Mock test correctly --- tests/optimize/test_hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index ec911c113..c57ff6852 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -843,6 +843,8 @@ def test_clean_hyperopt(mocker, hyperopt_conf, caplog): def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( From 6e70ae6e95758e0b5de881d956a81fbb88b0b0be Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 19:40:55 +0200 Subject: [PATCH 08/12] Improve code quality --- freqtrade/commands/hyperopt_commands.py | 1 - freqtrade/data/btanalysis.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 9075d19d2..fb567321d 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -86,7 +86,6 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: config['user_data_dir'] / 'hyperopt_results', config.get('hyperoptexportfilename')) - n = config.get('hyperopt_show_index', -1) filteroptions = { diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index d4edd41a8..b309a7634 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -91,7 +91,8 @@ def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str = * `directory/.last_result.json` does not exist * `directory/.last_result.json` has the wrong content """ - + if isinstance(directory, str): + directory = Path(directory) if predef_filename: return directory / predef_filename return directory / get_latest_hyperopt_filename(directory) From f3de74f817ef9e5c0a6f3cecd327c2ffa9018c75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Sep 2020 19:48:11 +0200 Subject: [PATCH 09/12] Mock all occurances of hyperopt.dump --- tests/optimize/test_hyperopt.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index c57ff6852..b889114e1 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -86,6 +86,7 @@ def create_results(mocker, hyperopt, testdatadir) -> List[Dict]: mocker.patch.object(Path, "unlink", MagicMock(return_value=True)) mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') return [{'loss': 1, 'result': 'foo', 'params': {}}] @@ -538,6 +539,8 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -900,6 +903,7 @@ def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -947,6 +951,7 @@ def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None: def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -993,6 +998,7 @@ def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1045,6 +1051,7 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1071,6 +1078,7 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1123,6 +1131,7 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1181,6 +1190,7 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: ]) def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( From 15bb0af1b354ddb197f081bb8b456269dc73841d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Sep 2020 00:35:19 +0200 Subject: [PATCH 10/12] Add some test-coverage --- tests/data/test_btanalysis.py | 3 +++ tests/optimize/test_hyperopt.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 43b1a7a4b..62d0415f2 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -51,6 +51,9 @@ def test_get_latest_hyperopt_file(testdatadir, mocker): res = get_latest_hyperopt_file(testdatadir.parent) assert res == testdatadir.parent / "hyperopt_results.pickle" + res = get_latest_hyperopt_file(str(testdatadir.parent)) + assert res == testdatadir.parent / "hyperopt_results.pickle" + def test_load_backtest_data_old_format(testdatadir): diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index b889114e1..ce23af0d8 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -5,7 +5,7 @@ from datetime import datetime from pathlib import Path from copy import deepcopy from typing import Dict, List -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock import pandas as pd import pytest @@ -81,14 +81,14 @@ def create_results(mocker, hyperopt, testdatadir) -> List[Dict]: mocker.patch.object(Path, "is_file", MagicMock(return_value=False)) stat_mock = MagicMock() - stat_mock.st_size = PropertyMock(return_value=1) - mocker.patch.object(Path, "stat", MagicMock(return_value=False)) + stat_mock.st_size = 1 + mocker.patch.object(Path, "stat", MagicMock(return_value=stat_mock)) mocker.patch.object(Path, "unlink", MagicMock(return_value=True)) mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') - return [{'loss': 1, 'result': 'foo', 'params': {}}] + return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}] def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None: @@ -524,6 +524,28 @@ def test_read_results_returns_epochs(mocker, hyperopt, testdatadir, caplog) -> N mock_load.assert_called_once() +def test_load_previous_results(mocker, hyperopt, testdatadir, caplog) -> None: + epochs = create_results(mocker, hyperopt, testdatadir) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) + mocker.patch.object(Path, 'is_file', MagicMock(return_value=True)) + statmock = MagicMock() + statmock.st_size = 5 + # mocker.patch.object(Path, 'stat', MagicMock(return_value=statmock)) + + results_file = testdatadir / 'optimize' / 'ut_results.pickle' + + hyperopt_epochs = hyperopt.load_previous_results(results_file) + + assert hyperopt_epochs == epochs + mock_load.assert_called_once() + + del epochs[0]['is_best'] + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) + + with pytest.raises(OperationalException): + hyperopt.load_previous_results(results_file) + + def test_roi_table_generation(hyperopt) -> None: params = { 'roi_t1': 5, From 17e605e130f9306d06987b685148032efd7b0f0e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Sep 2020 15:22:06 +0200 Subject: [PATCH 11/12] Make it clear in samples that strategy is mandatory --- docs/advanced-hyperopt.md | 4 ++-- docs/faq.md | 8 +------- docs/hyperopt.md | 4 ++-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 5fc674b03..dfabf2b91 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -27,9 +27,9 @@ class MyAwesomeHyperOpt2(MyAwesomeHyperOpt): and then quickly switch between hyperopt classes, running optimization process with hyperopt class you need in each particular case: ``` -$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt ... +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt --strategy MyAwesomeStrategy ... or -$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 ... +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 --strategy MyAwesomeStrategy ... ``` ## Creating and using a custom loss function diff --git a/docs/faq.md b/docs/faq.md index beed89801..d8af7798c 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -140,13 +140,7 @@ Since hyperopt uses Bayesian search, running for too many epochs may not produce It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. ```bash -freqtrade hyperopt -e 1000 -``` - -or if you want intermediate result to see - -```bash -for i in {1..100}; do freqtrade hyperopt -e 1000; done +freqtrade hyperopt --hyperop SampleHyperopt --strategy SampleStrategy -e 1000 ``` ### Why does it take a long time to run hyperopt? diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8bfd2cf7d..d26cbeeb2 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -229,7 +229,7 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -freqtrade hyperopt --config config.json --hyperopt -e 500 --spaces all +freqtrade hyperopt --config config.json --hyperopt --strategy -e 500 --spaces all ``` Use `` as the name of the custom hyperopt used. @@ -255,7 +255,7 @@ Use the `--timerange` argument to change how much of the test-set you want to us For example, to use one month of data, pass the following parameter to the hyperopt call: ```bash -freqtrade hyperopt --timerange 20180401-20180501 +freqtrade hyperopt --hyperopt --strategy --timerange 20180401-20180501 ``` ### Running Hyperopt using methods from a strategy From 0ea56548e4a435b403fb7ba48c48c7f433a0e4d4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Sep 2020 19:50:22 +0200 Subject: [PATCH 12/12] Try fix random test failure --- tests/rpc/test_rpc_telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3958a825a..400e33c6b 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1139,9 +1139,9 @@ def test_telegram_logs(default_conf, update, mocker) -> None: context = MagicMock() context.args = [] telegram._logs(update=update, context=context) - # Called at least 3 times. Exact times will change with unrelated changes to setup messages + # Called at least 2 times. Exact times will change with unrelated changes to setup messages # Therefore we don't test for this explicitly. - assert msg_mock.call_count > 3 + assert msg_mock.call_count >= 2 def test_edge_disabled(default_conf, update, mocker) -> None: