From 7a4edb1cd8f0c397bd16b19bcb67c4e0b7f40fd9 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sat, 29 Feb 2020 23:41:59 +0100 Subject: [PATCH 01/10] Fix: When total epochs is less than cpu cores --- freqtrade/optimize/hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 66ea18bd1..ee3f4d2c4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -557,6 +557,8 @@ class Hyperopt: cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) + if self.total_epochs < cpus: + config_jobs = self.total_epochs logger.info(f'Number of parallel jobs set as: {config_jobs}') self.dimensions: List[Dimension] = self.hyperopt_space() From e89fd33229e0b8c6e9ebbd8e49920205fcf0a81c Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sat, 29 Feb 2020 23:57:15 +0100 Subject: [PATCH 02/10] Fix for more arguments --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ee3f4d2c4..6b334507a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -557,7 +557,7 @@ class Hyperopt: cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) - if self.total_epochs < cpus: + if self.total_epochs < cpus and (config_jobs > self.total_epochs or config_jobs < 0): config_jobs = self.total_epochs logger.info(f'Number of parallel jobs set as: {config_jobs}') From 75b4f1a442e29084c4a93bda81419fe4223cd33d Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sun, 1 Mar 2020 14:12:27 +0100 Subject: [PATCH 03/10] Fix alignment of higher values --- freqtrade/optimize/hyperopt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 6b334507a..947aa78bf 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,6 +9,7 @@ import logging import random import sys import warnings +from math import ceil, floor from collections import OrderedDict from operator import itemgetter from pathlib import Path @@ -571,7 +572,7 @@ class Hyperopt: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() logger.info(f'Effective number of parallel workers used: {jobs}') - EVALS = max(self.total_epochs // jobs, 1) + EVALS = ceil(self.total_epochs / jobs) for i in range(EVALS): asked = self.opt.ask(n_points=jobs) f_val = self.run_optimizer_parallel(parallel, asked, i) @@ -580,6 +581,8 @@ class Hyperopt: for j in range(jobs): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 + if current > self.total_epochs: + continue val = f_val[j] val['current_epoch'] = current val['is_initial_point'] = current <= INITIAL_POINTS From f08c7eedf1b583a65e01d4ea5f30ead17f7bfad3 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sun, 1 Mar 2020 14:35:13 +0100 Subject: [PATCH 04/10] Changed jobs to be dynamic for last loop --- freqtrade/optimize/hyperopt.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 947aa78bf..91507c347 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,7 +9,7 @@ import logging import random import sys import warnings -from math import ceil, floor +from math import ceil from collections import OrderedDict from operator import itemgetter from pathlib import Path @@ -578,11 +578,13 @@ class Hyperopt: f_val = self.run_optimizer_parallel(parallel, asked, i) self.opt.tell(asked, [v['loss'] for v in f_val]) self.fix_optimizer_models_list() - for j in range(jobs): + if (i * jobs + jobs) > self.total_epochs: + current_jobs = jobs - ((i * jobs + jobs) - self.total_epochs) + else: + current_jobs = jobs + for j in range(current_jobs): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 - if current > self.total_epochs: - continue val = f_val[j] val['current_epoch'] = current val['is_initial_point'] = current <= INITIAL_POINTS From 7713cfeb7996b478f20c65afab9a1b49318b062b Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Mon, 2 Mar 2020 21:02:32 +0100 Subject: [PATCH 05/10] Corrected logic for -j + and - argument --- freqtrade/optimize/hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 91507c347..c500e71bf 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -558,7 +558,8 @@ class Hyperopt: cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) - if self.total_epochs < cpus and (config_jobs > self.total_epochs or config_jobs < 0): + if (config_jobs < 0 and (cpus + config_jobs + 1) > self.total_epochs) \ + or (config_jobs > 0 and config_jobs > self.total_epochs): config_jobs = self.total_epochs logger.info(f'Number of parallel jobs set as: {config_jobs}') From 0e4862b0c8f8cc927675eb4e3fb098f479f1d0b9 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Mon, 2 Mar 2020 22:58:54 +0100 Subject: [PATCH 06/10] Added logging if argument is miss-configured --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c500e71bf..98c19f15b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -561,6 +561,7 @@ class Hyperopt: if (config_jobs < 0 and (cpus + config_jobs + 1) > self.total_epochs) \ or (config_jobs > 0 and config_jobs > self.total_epochs): config_jobs = self.total_epochs + logger.info("Job count invalid will correct") logger.info(f'Number of parallel jobs set as: {config_jobs}') self.dimensions: List[Dimension] = self.hyperopt_space() From 92425642da922740e319f9563c93a36873af946f Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Tue, 3 Mar 2020 01:00:24 +0300 Subject: [PATCH 07/10] Fix config_jobs --- freqtrade/optimize/hyperopt.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 98c19f15b..66ce94ae0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -558,10 +558,6 @@ class Hyperopt: cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) - if (config_jobs < 0 and (cpus + config_jobs + 1) > self.total_epochs) \ - or (config_jobs > 0 and config_jobs > self.total_epochs): - config_jobs = self.total_epochs - logger.info("Job count invalid will correct") logger.info(f'Number of parallel jobs set as: {config_jobs}') self.dimensions: List[Dimension] = self.hyperopt_space() From a7d4755859740a62cc87dc7f168eb2f7fbb54f81 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Tue, 3 Mar 2020 01:20:14 +0300 Subject: [PATCH 08/10] optimize calculation of current_jobs --- freqtrade/optimize/hyperopt.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 66ce94ae0..27fb7bcdb 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -576,10 +576,12 @@ class Hyperopt: f_val = self.run_optimizer_parallel(parallel, asked, i) self.opt.tell(asked, [v['loss'] for v in f_val]) self.fix_optimizer_models_list() - if (i * jobs + jobs) > self.total_epochs: - current_jobs = jobs - ((i * jobs + jobs) - self.total_epochs) - else: - current_jobs = jobs + + # Correct the number of epochs to handled for the last + # iteration (should not exceed self.total_epochs) + n_rest = (i + 1) * jobs - self.total_epochs + current_jobs = jobs - n_rest if n_rest > 0 else jobs + for j in range(current_jobs): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 From 45c94967927609b556f3b5ec226cc5dc31d9a8a9 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Tue, 3 Mar 2020 01:33:11 +0300 Subject: [PATCH 09/10] Do not run optimizer for 'jobs' epochs for the last iteration --- freqtrade/optimize/hyperopt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 27fb7bcdb..41e2ce82b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -572,16 +572,16 @@ class Hyperopt: logger.info(f'Effective number of parallel workers used: {jobs}') EVALS = ceil(self.total_epochs / jobs) for i in range(EVALS): - asked = self.opt.ask(n_points=jobs) + # Correct the number of epochs to be processed for the last + # iteration (should not exceed self.total_epochs in total) + n_rest = (i + 1) * jobs - self.total_epochs + current_jobs = jobs - n_rest if n_rest > 0 else jobs + + asked = self.opt.ask(n_points=current_jobs) f_val = self.run_optimizer_parallel(parallel, asked, i) self.opt.tell(asked, [v['loss'] for v in f_val]) self.fix_optimizer_models_list() - # Correct the number of epochs to handled for the last - # iteration (should not exceed self.total_epochs) - n_rest = (i + 1) * jobs - self.total_epochs - current_jobs = jobs - n_rest if n_rest > 0 else jobs - for j in range(current_jobs): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 From 52cd5f912789d7b151702ba6c0a7caf3e4414846 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Tue, 3 Mar 2020 01:42:25 +0300 Subject: [PATCH 10/10] Better use enumerate: more correct and more pythonic --- freqtrade/optimize/hyperopt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 41e2ce82b..b3be3f160 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -582,10 +582,9 @@ class Hyperopt: self.opt.tell(asked, [v['loss'] for v in f_val]) self.fix_optimizer_models_list() - for j in range(current_jobs): + for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 - val = f_val[j] val['current_epoch'] = current val['is_initial_point'] = current <= INITIAL_POINTS logger.debug(f"Optimizer epoch evaluated: {val}")