Changed back to progressbar2 for better handling of logger.

Coloring still needs some work (bug + what colors to use)
This commit is contained in:
Fredrik81 2020-04-06 13:12:32 +02:00
parent 5737139979
commit d5609d4997
3 changed files with 59 additions and 51 deletions

View File

@ -22,7 +22,7 @@ from colorama import init as colorama_init
from joblib import (Parallel, cpu_count, delayed, dump, load,
wrap_non_picklable_objects)
from pandas import DataFrame, json_normalize, isna
from tqdm import tqdm
import progressbar
import tabulate
from os import path
import io
@ -44,7 +44,8 @@ with warnings.catch_warnings():
from skopt import Optimizer
from skopt.space import Dimension
progressbar.streams.wrap_stderr()
progressbar.streams.wrap_stdout()
logger = logging.getLogger(__name__)
@ -682,12 +683,27 @@ class Hyperopt:
logger.info(f'Effective number of parallel workers used: {jobs}')
# Define progressbar
self.progress_bar = tqdm(
total=self.total_epochs, ncols=108, unit=' Epoch',
bar_format='Epoch {n_fmt}/{total_fmt} ({percentage:3.0f}%)|{bar}|'
' [{elapsed}<{remaining} {rate_fmt}{postfix}]'
)
if self.print_colorized:
widgets = [
' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
' (', progressbar.Percentage(), ')] ',
progressbar.Bar(marker=progressbar.AnimatedMarker(
fill='',
fill_wrap='\x1b[32m{}\x1b[39m',
marker_wrap='\x1b[31m{}\x1b[39m',
)),
' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
]
else:
widgets = [
' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), '] ',
progressbar.Bar(marker=''),
' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
]
with progressbar.ProgressBar(
maxval=self.total_epochs, redirect_stdout=True, redirect_stderr=True,
widgets=widgets
) as pbar:
EVALS = ceil(self.total_epochs / jobs)
for i in range(EVALS):
# Correct the number of epochs to be processed for the last
@ -706,6 +722,7 @@ class Hyperopt:
current = i * jobs + j + 1
val['current_epoch'] = current
val['is_initial_point'] = current <= INITIAL_POINTS
logger.debug(f"Optimizer epoch evaluated: {val}")
is_best = self.is_best_loss(val, self.current_best_loss)
@ -714,23 +731,19 @@ class Hyperopt:
# evaluations can take different time. Here they are aligned in the
# order they will be shown to the user.
val['is_best'] = is_best
output = self.get_results(val)
if output:
self.progress_bar.write(output)
self.progress_bar.ncols = 108
self.progress_bar.update(1)
self.print_results(val)
if is_best:
self.current_best_loss = val['loss']
self.trials.append(val)
# Save results after each best epoch and every 100 epochs
if is_best or current % 100 == 0:
self.save_trials()
self.progress_bar.ncols = 108
self.progress_bar.close()
pbar.update(current)
except KeyboardInterrupt:
self.progress_bar.close()
print('User interrupted..')
self.save_trials(final=True)
@ -743,9 +756,3 @@ class Hyperopt:
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.
print("No epochs evaluated yet, no best result.")
def __getstate__(self):
state = self.__dict__.copy()
del state['trials']
del state['progress_bar']
return state

View File

@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1
scikit-optimize==0.7.4
filelock==3.0.12
joblib==0.14.1
tqdm==4.43.0
progressbar2==3.50.1

View File

@ -24,6 +24,7 @@ hyperopt = [
'scikit-optimize',
'filelock',
'joblib',
'progressbar2',
]
develop = [