From 3a8b68c0fd721094c606286b4dce6a86d89442d9 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Tue, 10 Mar 2020 20:30:36 +0100 Subject: [PATCH 01/14] Initial work on progressbar --- freqtrade/loggers.py | 9 ++++++--- freqtrade/optimize/hyperopt.py | 34 ++++++++++++++++++++++++++++++---- requirements-hyperopt.txt | 1 + 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index c69388430..609604ff2 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -4,6 +4,7 @@ import sys from logging import Formatter from logging.handlers import RotatingFileHandler, SysLogHandler from typing import Any, Dict, List +import progressbar from freqtrade.exceptions import OperationalException @@ -18,13 +19,13 @@ def _set_loggers(verbosity: int = 0) -> None: """ logging.getLogger('requests').setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG + logging.INFO if verbosity <= 1 else logging.DEBUG ) logging.getLogger("urllib3").setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG + logging.INFO if verbosity <= 1 else logging.DEBUG ) logging.getLogger('ccxt.base.exchange').setLevel( - logging.INFO if verbosity <= 2 else logging.DEBUG + logging.INFO if verbosity <= 2 else logging.DEBUG ) logging.getLogger('telegram').setLevel(logging.INFO) @@ -36,6 +37,8 @@ def setup_logging(config: Dict[str, Any]) -> None: # Log level verbosity = config['verbosity'] + progressbar.streams.wrap_stderr() + # Log to stderr log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)] diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4c32a0543..93fc3c558 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,7 +7,6 @@ This module contains the hyperopt logic import locale import logging import random -import sys import warnings from math import ceil from collections import OrderedDict @@ -22,8 +21,9 @@ from colorama import init as colorama_init from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) from pandas import DataFrame, json_normalize, isna +import progressbar import tabulate -from os import path +from os import path, popen import io from freqtrade.data.converter import trim_dataframe @@ -270,7 +270,6 @@ class Hyperopt: # Print '\n' after each 100th epoch to separate dots from the log messages. # Otherwise output is messy on a terminal. print('.', end='' if results['current_epoch'] % 100 != 0 else None) # type: ignore - sys.stdout.flush() if self.print_all or is_best: if not self.print_all: @@ -622,6 +621,10 @@ class Hyperopt: def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) + def _get_height(self) -> int: + rows = int((popen('stty size', 'r').read().split())[0]) + return rows + def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) logger.info(f"Using optimizer random state: {self.random_state}") @@ -629,7 +632,6 @@ class Hyperopt: data, timerange = self.backtesting.load_bt_data() preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) - # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = trim_dataframe(df, timerange) @@ -659,6 +661,14 @@ class Hyperopt: try: with Parallel(n_jobs=config_jobs) as parallel: + self.progress_bar = progressbar.ProgressBar( + min_value=0, + max_value=self.total_epochs, + initial_value=0, + line_breaks=True, + enable_colors=self.print_colorized + ) + self.progress_bar.start() jobs = parallel._effective_n_jobs() logger.info(f'Effective number of parallel workers used: {jobs}') EVALS = ceil(self.total_epochs / jobs) @@ -673,6 +683,9 @@ class Hyperopt: self.opt.tell(asked, [v['loss'] for v in f_val]) self.fix_optimizer_models_list() + # Calculate progressbar outputs + pbar_line = ceil(self._get_height() / 2) + for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 @@ -689,12 +702,19 @@ class Hyperopt: self.print_results(val) + if pbar_line <= current: + self.progress_bar.update(current) + pbar_line = current + ceil(self._get_height() / 2) + if is_best: self.current_best_loss = val['loss'] self.trials.append(val) # Save results after each best epoch and every 100 epochs if is_best or current % 100 == 0: self.save_trials() + self.progress_bar.finish() + + # self.progress_bar.update(current) except KeyboardInterrupt: print('User interrupted..') @@ -708,3 +728,9 @@ class Hyperopt: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. print("No epochs evaluated yet, no best result.") + + def __getstate__(self): + state = self.__dict__.copy() + del state['trials'] + del state['progress_bar'] + return state diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index c7e586a33..3c500839e 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -7,3 +7,4 @@ scikit-learn==0.22.2.post1 scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 +progressbar2==3.50.0 From 81cbb925568dbae6260caf88ced5c90f7abc1a3b Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Wed, 11 Mar 2020 22:30:36 +0100 Subject: [PATCH 02/14] Switch to TQDM --- freqtrade/commands/hyperopt_commands.py | 4 +- freqtrade/optimize/hyperopt.py | 71 +++++++++++++++++++------ 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 5b2388252..3f61ea66c 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -52,8 +52,8 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if not export_csv: try: - Hyperopt.print_result_table(config, trials, total_epochs, - not filteroptions['only_best'], print_colorized, 0) + print(Hyperopt.get_result_table(config, trials, total_epochs, + not filteroptions['only_best'], print_colorized, 0)) except KeyboardInterrupt: print('User interrupted..') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 93fc3c558..e110f7e9c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -21,7 +21,7 @@ from colorama import init as colorama_init from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) from pandas import DataFrame, json_normalize, isna -import progressbar +from tqdm import tqdm import tabulate from os import path, popen import io @@ -275,11 +275,37 @@ class Hyperopt: if not self.print_all: # Separate the results explanation string from dots print("\n") - self.print_result_table(self.config, results, self.total_epochs, - self.print_all, self.print_colorized, - self.hyperopt_table_header) + print(self.get_result_table( + self.config, results, self.total_epochs, + self.print_all, self.print_colorized, + self.hyperopt_table_header + ) + ) self.hyperopt_table_header = 2 + def get_results(self, results) -> str: + """ + Log results if it is better than any previous evaluation + """ + output = '' + is_best = results['is_best'] + # if not self.print_all: + # Print '\n' after each 100th epoch to separate dots from the log messages. + # Otherwise output is messy on a terminal. + # return '.', end='' if results['current_epoch'] % 100 != 0 else None # type: ignore + + if self.print_all or is_best: + # if not self.print_all: + # Separate the results explanation string from dots + # print("\n") + output = self.get_result_table( + self.config, results, self.total_epochs, + self.print_all, self.print_colorized, + self.hyperopt_table_header + ) + self.hyperopt_table_header = 2 + return output + @staticmethod def print_results_explanation(results, total_epochs, highlight_best: bool, print_colorized: bool) -> None: @@ -303,13 +329,13 @@ class Hyperopt: f"Objective: {results['loss']:.5f}") @staticmethod - def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, - print_colorized: bool, remove_header: int) -> None: + def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, + print_colorized: bool, remove_header: int) -> str: """ Log result table """ if not results: - return + return '' tabulate.PRESERVE_WHITESPACE = True @@ -380,7 +406,7 @@ class Hyperopt: trials.to_dict(orient='list'), tablefmt='psql', headers='keys', stralign="right" ) - print(table) + return table @staticmethod def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool, @@ -661,6 +687,7 @@ class Hyperopt: try: with Parallel(n_jobs=config_jobs) as parallel: + """ self.progress_bar = progressbar.ProgressBar( min_value=0, max_value=self.total_epochs, @@ -668,9 +695,17 @@ class Hyperopt: line_breaks=True, enable_colors=self.print_colorized ) - self.progress_bar.start() + """ jobs = parallel._effective_n_jobs() logger.info(f'Effective number of parallel workers used: {jobs}') + + # Define progressbar + self.progress_bar = tqdm( + total=self.total_epochs, ncols=108, unit=' Epoch', + bar_format='Epoch {n_fmt}/{total_fmt} ({percentage:3.0f}%)|{bar}|' + ' [{elapsed}<{remaining} {rate_fmt}{postfix}]' + ) + EVALS = ceil(self.total_epochs / jobs) for i in range(EVALS): # Correct the number of epochs to be processed for the last @@ -684,8 +719,7 @@ class Hyperopt: self.fix_optimizer_models_list() # Calculate progressbar outputs - pbar_line = ceil(self._get_height() / 2) - + # pbar_line = ceil(self._get_height() / 2) for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 @@ -699,20 +733,25 @@ class Hyperopt: # evaluations can take different time. Here they are aligned in the # order they will be shown to the user. val['is_best'] = is_best - - self.print_results(val) - + # print(current) + output = self.get_results(val) + self.progress_bar.write(output) + # self.progress_bar.write(str(len(output.split('\n')[0]))) + self.progress_bar.ncols = len(output.split('\n')[0]) + self.progress_bar.update(1) + """ if pbar_line <= current: self.progress_bar.update(current) pbar_line = current + ceil(self._get_height() / 2) - + """ if is_best: self.current_best_loss = val['loss'] self.trials.append(val) # Save results after each best epoch and every 100 epochs if is_best or current % 100 == 0: self.save_trials() - self.progress_bar.finish() + self.progress_bar.ncols = 108 + self.progress_bar.close() # self.progress_bar.update(current) except KeyboardInterrupt: From 755763ec42b51370c7326e4e90f5c35921ba2125 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Wed, 11 Mar 2020 22:43:27 +0100 Subject: [PATCH 03/14] Update requirements --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 3c500839e..7469674cd 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1 scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 -progressbar2==3.50.0 +tqdm==4.43.0 From 40a413c524b9eb2b2d13e222ac1e81173a1d5b1e Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Wed, 11 Mar 2020 22:50:23 +0100 Subject: [PATCH 04/14] More remove of progressbar2 --- freqtrade/loggers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index 609604ff2..153ce8c80 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -4,7 +4,6 @@ import sys from logging import Formatter from logging.handlers import RotatingFileHandler, SysLogHandler from typing import Any, Dict, List -import progressbar from freqtrade.exceptions import OperationalException @@ -37,8 +36,6 @@ def setup_logging(config: Dict[str, Any]) -> None: # Log level verbosity = config['verbosity'] - progressbar.streams.wrap_stderr() - # Log to stderr log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)] From 9387ed923c97a2dffebfa1ca57662745d97a0f44 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 12 Mar 2020 02:07:50 +0100 Subject: [PATCH 05/14] fix for empty lines --- freqtrade/optimize/hyperopt.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e110f7e9c..48a89e1d6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -289,15 +289,8 @@ class Hyperopt: """ output = '' is_best = results['is_best'] - # if not self.print_all: - # Print '\n' after each 100th epoch to separate dots from the log messages. - # Otherwise output is messy on a terminal. - # return '.', end='' if results['current_epoch'] % 100 != 0 else None # type: ignore if self.print_all or is_best: - # if not self.print_all: - # Separate the results explanation string from dots - # print("\n") output = self.get_result_table( self.config, results, self.total_epochs, self.print_all, self.print_colorized, @@ -735,9 +728,10 @@ class Hyperopt: val['is_best'] = is_best # print(current) output = self.get_results(val) - self.progress_bar.write(output) + if output: + self.progress_bar.write(output) # self.progress_bar.write(str(len(output.split('\n')[0]))) - self.progress_bar.ncols = len(output.split('\n')[0]) + self.progress_bar.ncols = 108 self.progress_bar.update(1) """ if pbar_line <= current: From df1ae565dc09c93b046b2ba79cb5a1ce647c1d09 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 12 Mar 2020 02:26:41 +0100 Subject: [PATCH 06/14] clean-up --- freqtrade/optimize/hyperopt.py | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 48a89e1d6..a6da8c4b8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,7 +23,7 @@ from joblib import (Parallel, cpu_count, delayed, dump, load, from pandas import DataFrame, json_normalize, isna from tqdm import tqdm import tabulate -from os import path, popen +from os import path import io from freqtrade.data.converter import trim_dataframe @@ -640,10 +640,6 @@ class Hyperopt: def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) - def _get_height(self) -> int: - rows = int((popen('stty size', 'r').read().split())[0]) - return rows - def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) logger.info(f"Using optimizer random state: {self.random_state}") @@ -680,15 +676,6 @@ class Hyperopt: try: with Parallel(n_jobs=config_jobs) as parallel: - """ - self.progress_bar = progressbar.ProgressBar( - min_value=0, - max_value=self.total_epochs, - initial_value=0, - line_breaks=True, - enable_colors=self.print_colorized - ) - """ jobs = parallel._effective_n_jobs() logger.info(f'Effective number of parallel workers used: {jobs}') @@ -712,7 +699,6 @@ class Hyperopt: self.fix_optimizer_models_list() # Calculate progressbar outputs - # pbar_line = ceil(self._get_height() / 2) for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 @@ -726,18 +712,12 @@ class Hyperopt: # evaluations can take different time. Here they are aligned in the # order they will be shown to the user. val['is_best'] = is_best - # print(current) output = self.get_results(val) if output: self.progress_bar.write(output) - # self.progress_bar.write(str(len(output.split('\n')[0]))) self.progress_bar.ncols = 108 self.progress_bar.update(1) - """ - if pbar_line <= current: - self.progress_bar.update(current) - pbar_line = current + ceil(self._get_height() / 2) - """ + if is_best: self.current_best_loss = val['loss'] self.trials.append(val) @@ -747,8 +727,8 @@ class Hyperopt: self.progress_bar.ncols = 108 self.progress_bar.close() - # self.progress_bar.update(current) except KeyboardInterrupt: + self.progress_bar.close() print('User interrupted..') self.save_trials(final=True) @@ -761,9 +741,3 @@ class Hyperopt: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. print("No epochs evaluated yet, no best result.") - - def __getstate__(self): - state = self.__dict__.copy() - del state['trials'] - del state['progress_bar'] - return state From 1a59fc11be8312c19cd5723f641ac66128b05117 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 12 Mar 2020 02:36:18 +0100 Subject: [PATCH 07/14] doh --- freqtrade/optimize/hyperopt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a6da8c4b8..9ff4a1ead 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -741,3 +741,9 @@ class Hyperopt: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. print("No epochs evaluated yet, no best result.") + + def __getstate__(self): + state = self.__dict__.copy() + del state['trials'] + del state['progress_bar'] + return state From 5737139979351eab8482cbe230b111310e11981b Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 12 Mar 2020 16:47:09 +0100 Subject: [PATCH 08/14] Small fix --- freqtrade/optimize/hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 9ff4a1ead..a5921703f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,6 +7,7 @@ This module contains the hyperopt logic import locale import logging import random +import sys import warnings from math import ceil from collections import OrderedDict @@ -270,6 +271,7 @@ class Hyperopt: # Print '\n' after each 100th epoch to separate dots from the log messages. # Otherwise output is messy on a terminal. print('.', end='' if results['current_epoch'] % 100 != 0 else None) # type: ignore + sys.stdout.flush() if self.print_all or is_best: if not self.print_all: From d5609d49972f2d8ddc873b945090bba2635db00f Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Mon, 6 Apr 2020 13:12:32 +0200 Subject: [PATCH 09/14] Changed back to progressbar2 for better handling of logger. Coloring still needs some work (bug + what colors to use) --- freqtrade/optimize/hyperopt.py | 107 ++++++++++++++++++--------------- requirements-hyperopt.txt | 2 +- setup.py | 1 + 3 files changed, 59 insertions(+), 51 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a5921703f..1adaf54e1 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,7 @@ from colorama import init as colorama_init from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) from pandas import DataFrame, json_normalize, isna -from tqdm import tqdm +import progressbar import tabulate from os import path import io @@ -44,7 +44,8 @@ with warnings.catch_warnings(): from skopt import Optimizer from skopt.space import Dimension - +progressbar.streams.wrap_stderr() +progressbar.streams.wrap_stdout() logger = logging.getLogger(__name__) @@ -682,55 +683,67 @@ class Hyperopt: logger.info(f'Effective number of parallel workers used: {jobs}') # Define progressbar - self.progress_bar = tqdm( - total=self.total_epochs, ncols=108, unit=' Epoch', - bar_format='Epoch {n_fmt}/{total_fmt} ({percentage:3.0f}%)|{bar}|' - ' [{elapsed}<{remaining} {rate_fmt}{postfix}]' - ) + if self.print_colorized: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='█', + fill_wrap='\x1b[32m{}\x1b[39m', + marker_wrap='\x1b[31m{}\x1b[39m', + )), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + else: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), '] ', + progressbar.Bar(marker='█'), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + with progressbar.ProgressBar( + maxval=self.total_epochs, redirect_stdout=True, redirect_stderr=True, + widgets=widgets + ) as pbar: + EVALS = ceil(self.total_epochs / jobs) + for i in range(EVALS): + # Correct the number of epochs to be processed for the last + # iteration (should not exceed self.total_epochs in total) + n_rest = (i + 1) * jobs - self.total_epochs + current_jobs = jobs - n_rest if n_rest > 0 else jobs - EVALS = ceil(self.total_epochs / jobs) - for i in range(EVALS): - # Correct the number of epochs to be processed for the last - # iteration (should not exceed self.total_epochs in total) - n_rest = (i + 1) * jobs - self.total_epochs - current_jobs = jobs - n_rest if n_rest > 0 else jobs + asked = self.opt.ask(n_points=current_jobs) + f_val = self.run_optimizer_parallel(parallel, asked, i) + self.opt.tell(asked, [v['loss'] for v in f_val]) + self.fix_optimizer_models_list() - asked = self.opt.ask(n_points=current_jobs) - f_val = self.run_optimizer_parallel(parallel, asked, i) - self.opt.tell(asked, [v['loss'] for v in f_val]) - self.fix_optimizer_models_list() + # Calculate progressbar outputs + for j, val in enumerate(f_val): + # Use human-friendly indexes here (starting from 1) + current = i * jobs + j + 1 + val['current_epoch'] = current + val['is_initial_point'] = current <= INITIAL_POINTS - # Calculate progressbar outputs - for j, val in enumerate(f_val): - # Use human-friendly indexes here (starting from 1) - current = i * jobs + j + 1 - val['current_epoch'] = current - val['is_initial_point'] = current <= INITIAL_POINTS - logger.debug(f"Optimizer epoch evaluated: {val}") + logger.debug(f"Optimizer epoch evaluated: {val}") - is_best = self.is_best_loss(val, self.current_best_loss) - # This value is assigned here and not in the optimization method - # to keep proper order in the list of results. That's because - # evaluations can take different time. Here they are aligned in the - # order they will be shown to the user. - val['is_best'] = is_best - output = self.get_results(val) - if output: - self.progress_bar.write(output) - self.progress_bar.ncols = 108 - self.progress_bar.update(1) + is_best = self.is_best_loss(val, self.current_best_loss) + # This value is assigned here and not in the optimization method + # to keep proper order in the list of results. That's because + # evaluations can take different time. Here they are aligned in the + # order they will be shown to the user. + val['is_best'] = is_best + self.print_results(val) - if is_best: - self.current_best_loss = val['loss'] - self.trials.append(val) - # Save results after each best epoch and every 100 epochs - if is_best or current % 100 == 0: - self.save_trials() - self.progress_bar.ncols = 108 - self.progress_bar.close() + if is_best: + self.current_best_loss = val['loss'] + self.trials.append(val) + + # Save results after each best epoch and every 100 epochs + if is_best or current % 100 == 0: + self.save_trials() + + pbar.update(current) except KeyboardInterrupt: - self.progress_bar.close() print('User interrupted..') self.save_trials(final=True) @@ -743,9 +756,3 @@ class Hyperopt: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. print("No epochs evaluated yet, no best result.") - - def __getstate__(self): - state = self.__dict__.copy() - del state['trials'] - del state['progress_bar'] - return state diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 7469674cd..6df1eb157 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1 scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 -tqdm==4.43.0 +progressbar2==3.50.1 diff --git a/setup.py b/setup.py index 7890f862e..94c48a6a7 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ hyperopt = [ 'scikit-optimize', 'filelock', 'joblib', + 'progressbar2', ] develop = [ From c95906cfcf89210b7045fc9f95ace14aa6c2cc31 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Tue, 7 Apr 2020 10:42:15 +0200 Subject: [PATCH 10/14] Update hyperopt.py --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 1adaf54e1..a839d2111 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -649,7 +649,7 @@ class Hyperopt: self.hyperopt_table_header = -1 data, timerange = self.backtesting.load_bt_data() - preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) + preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data) # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = trim_dataframe(df, timerange) From 132f5f73f5bd7c083891802704e7bb208b12012b Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Tue, 7 Apr 2020 10:44:18 +0200 Subject: [PATCH 11/14] Update hyperopt.py --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a839d2111..bf32997c1 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -650,6 +650,7 @@ class Hyperopt: data, timerange = self.backtesting.load_bt_data() preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data) + # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = trim_dataframe(df, timerange) From 4707484a4cf8982bb9aa1548b5ba754356981d0d Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 9 Apr 2020 11:42:13 +0200 Subject: [PATCH 12/14] Fix issue with colring enabled + styling --- freqtrade/optimize/hyperopt.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index fac613549..f34967af2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,7 +7,6 @@ This module contains the hyperopt logic import locale import logging import random -import sys import warnings from math import ceil from collections import OrderedDict @@ -18,7 +17,6 @@ from typing import Any, Dict, List, Optional import rapidjson from colorama import Fore, Style -from colorama import init as colorama_init from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) from pandas import DataFrame, json_normalize, isna @@ -268,17 +266,10 @@ class Hyperopt: Log results if it is better than any previous evaluation """ is_best = results['is_best'] - if not self.print_all: - # Print '\n' after each 100th epoch to separate dots from the log messages. - # Otherwise output is messy on a terminal. - print('.', end='' if results['current_epoch'] % 100 != 0 else None) # type: ignore - sys.stdout.flush() if self.print_all or is_best: - if not self.print_all: - # Separate the results explanation string from dots - print("\n") - print(self.get_result_table( + print( + self.get_result_table( self.config, results, self.total_epochs, self.print_all, self.print_colorized, self.hyperopt_table_header @@ -675,9 +666,6 @@ class Hyperopt: self.dimensions: List[Dimension] = self.hyperopt_space() self.opt = self.get_optimizer(self.dimensions, config_jobs) - if self.print_colorized: - colorama_init(autoreset=True) - try: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() @@ -690,15 +678,18 @@ class Hyperopt: ' (', progressbar.Percentage(), ')] ', progressbar.Bar(marker=progressbar.AnimatedMarker( fill='█', - fill_wrap='\x1b[32m{}\x1b[39m', - marker_wrap='\x1b[31m{}\x1b[39m', + fill_wrap=Fore.GREEN + '{}' + Fore.RESET, + marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, )), ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', ] else: widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), '] ', - progressbar.Bar(marker='█'), + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='█', + )), ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', ] with progressbar.ProgressBar( From d9e54ab7a4b386a1faa89dbb995fceb1f1c90a92 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sat, 11 Apr 2020 17:42:19 +0200 Subject: [PATCH 13/14] Update freqtrade/optimize/hyperopt.py nice find Co-Authored-By: Matthias --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f34967af2..4b9139681 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -677,7 +677,7 @@ class Hyperopt: ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), ' (', progressbar.Percentage(), ')] ', progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='█', + fill='\N{FULL BLOCK}', fill_wrap=Fore.GREEN + '{}' + Fore.RESET, marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, )), From 2c1c1c7f166871d1ba7ce95803426c50f85ff4df Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sat, 11 Apr 2020 17:42:32 +0200 Subject: [PATCH 14/14] Update freqtrade/optimize/hyperopt.py nice find Co-Authored-By: Matthias --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4b9139681..affd6f3b3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -688,7 +688,7 @@ class Hyperopt: ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), ' (', progressbar.Percentage(), ')] ', progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='█', + fill='\N{FULL BLOCK}', )), ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', ]