From d532da90710df33752bca88959efa61fbfe456cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 16:04:31 +0200 Subject: [PATCH 1/9] Add Rich Progressbar Wrapper --- freqtrade/util/rich_progress.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 freqtrade/util/rich_progress.py diff --git a/freqtrade/util/rich_progress.py b/freqtrade/util/rich_progress.py new file mode 100644 index 000000000..7fe00b4d8 --- /dev/null +++ b/freqtrade/util/rich_progress.py @@ -0,0 +1,27 @@ +import logging +import sys +from contextlib import contextmanager + +from rich.progress import Progress + + +@contextmanager +def FtProgress(*args, **kwargs): + """ + Wrapper around rich.progress.Progress to fix issues with logging. + """ + try: + __logger = kwargs.pop('logger', None) + streamhandlers = [x for x in __logger.root.handlers if type(x) == logging.StreamHandler] + __prior_stderr = [] + + with Progress(*args, **kwargs) as progress: + for handler in streamhandlers: + __prior_stderr.append(handler.stream) + handler.setStream(sys.stderr) + + yield progress + + finally: + for idx, handler in enumerate(streamhandlers): + handler.setStream(__prior_stderr[idx]) From 40450ebeccb1d32f2b168d0d9da93999510bc263 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 16:05:10 +0200 Subject: [PATCH 2/9] Add dependency on Rich --- freqtrade/util/__init__.py | 1 + requirements.txt | 1 + setup.py | 1 + 3 files changed, 3 insertions(+) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 3c3c034c1..50527eb97 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,2 +1,3 @@ from freqtrade.util.ft_precise import FtPrecise # noqa: F401 from freqtrade.util.periodic_cache import PeriodicCache # noqa: F401 +from freqtrade.util.rich_progress import FtProgress # noqa: F401 diff --git a/requirements.txt b/requirements.txt index 34c7da0fa..f4f9b13fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ jinja2==3.1.2 tables==3.8.0 blosc==1.11.1 joblib==1.2.0 +rich==13.3.3 pyarrow==11.0.0; platform_machine != 'armv7l' # find first, C search in arrays diff --git a/setup.py b/setup.py index 131e8a8a7..5c95e9316 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ setup( 'numpy', 'pandas', 'joblib>=1.2.0', + 'rich', 'pyarrow; platform_machine != "armv7l"', 'fastapi', 'pydantic>=1.8.0', From b6aac5079beb3f4f06320664665426406f7a1ba1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 16:21:30 +0200 Subject: [PATCH 3/9] REmove Rich-progress wrapper again --- freqtrade/util/__init__.py | 1 - freqtrade/util/rich_progress.py | 27 --------------------------- 2 files changed, 28 deletions(-) delete mode 100644 freqtrade/util/rich_progress.py diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 50527eb97..3c3c034c1 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,3 +1,2 @@ from freqtrade.util.ft_precise import FtPrecise # noqa: F401 from freqtrade.util.periodic_cache import PeriodicCache # noqa: F401 -from freqtrade.util.rich_progress import FtProgress # noqa: F401 diff --git a/freqtrade/util/rich_progress.py b/freqtrade/util/rich_progress.py deleted file mode 100644 index 7fe00b4d8..000000000 --- a/freqtrade/util/rich_progress.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging -import sys -from contextlib import contextmanager - -from rich.progress import Progress - - -@contextmanager -def FtProgress(*args, **kwargs): - """ - Wrapper around rich.progress.Progress to fix issues with logging. - """ - try: - __logger = kwargs.pop('logger', None) - streamhandlers = [x for x in __logger.root.handlers if type(x) == logging.StreamHandler] - __prior_stderr = [] - - with Progress(*args, **kwargs) as progress: - for handler in streamhandlers: - __prior_stderr.append(handler.stream) - handler.setStream(sys.stderr) - - yield progress - - finally: - for idx, handler in enumerate(streamhandlers): - handler.setStream(__prior_stderr[idx]) From 818d18d4e0922ce70f5c0dd2797c9bfc38c3f926 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 16:23:00 +0200 Subject: [PATCH 4/9] Add StdErrStreamHandler to logging --- freqtrade/loggers.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index 823fa174e..a9ba7d7d8 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -1,12 +1,36 @@ import logging import sys -from logging import Formatter +from logging import Formatter, Handler from logging.handlers import BufferingHandler, RotatingFileHandler, SysLogHandler from freqtrade.constants import Config from freqtrade.exceptions import OperationalException +class FTStdErrStreamHandler(Handler): + def flush(self): + """ + Override Flush behaviour - we keep half of the configured capacity + otherwise, we have moments with "empty" logs. + """ + self.acquire() + try: + sys.stderr.flush() + finally: + self.release() + + def emit(self, record): + try: + msg = self.format(record) + # Don't keep a reference to stderr - this can be problematic with progressbars. + sys.stderr.write(msg + '\n') + self.flush() + except RecursionError: + raise + except Exception: + self.handleError(record) + + class FTBufferingHandler(BufferingHandler): def flush(self): """ @@ -69,7 +93,7 @@ def setup_logging_pre() -> None: logging.basicConfig( level=logging.INFO, format=LOGFORMAT, - handlers=[logging.StreamHandler(sys.stderr), bufferHandler] + handlers=[FTStdErrStreamHandler(), bufferHandler] ) From ed57e7d43bc8bd08ecb5ada5fd5cc6a0f8465e2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 16:48:18 +0200 Subject: [PATCH 5/9] Refactor logging to be a package, instead of a module --- freqtrade/{loggers.py => loggers/__init__.py} | 45 ++----------------- freqtrade/loggers/buffering_handler.py | 15 +++++++ freqtrade/loggers/std_err_stream_handler.py | 26 +++++++++++ 3 files changed, 45 insertions(+), 41 deletions(-) rename freqtrade/{loggers.py => loggers/__init__.py} (79%) create mode 100644 freqtrade/loggers/buffering_handler.py create mode 100644 freqtrade/loggers/std_err_stream_handler.py diff --git a/freqtrade/loggers.py b/freqtrade/loggers/__init__.py similarity index 79% rename from freqtrade/loggers.py rename to freqtrade/loggers/__init__.py index a9ba7d7d8..528d274f2 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers/__init__.py @@ -1,48 +1,11 @@ import logging -import sys -from logging import Formatter, Handler -from logging.handlers import BufferingHandler, RotatingFileHandler, SysLogHandler +from logging import Formatter +from logging.handlers import RotatingFileHandler, SysLogHandler from freqtrade.constants import Config from freqtrade.exceptions import OperationalException - - -class FTStdErrStreamHandler(Handler): - def flush(self): - """ - Override Flush behaviour - we keep half of the configured capacity - otherwise, we have moments with "empty" logs. - """ - self.acquire() - try: - sys.stderr.flush() - finally: - self.release() - - def emit(self, record): - try: - msg = self.format(record) - # Don't keep a reference to stderr - this can be problematic with progressbars. - sys.stderr.write(msg + '\n') - self.flush() - except RecursionError: - raise - except Exception: - self.handleError(record) - - -class FTBufferingHandler(BufferingHandler): - def flush(self): - """ - Override Flush behaviour - we keep half of the configured capacity - otherwise, we have moments with "empty" logs. - """ - self.acquire() - try: - # Keep half of the records in buffer. - self.buffer = self.buffer[-int(self.capacity / 2):] - finally: - self.release() +from freqtrade.loggers.buffering_handler import FTBufferingHandler +from freqtrade.loggers.std_err_stream_handler import FTStdErrStreamHandler logger = logging.getLogger(__name__) diff --git a/freqtrade/loggers/buffering_handler.py b/freqtrade/loggers/buffering_handler.py new file mode 100644 index 000000000..e4621fa79 --- /dev/null +++ b/freqtrade/loggers/buffering_handler.py @@ -0,0 +1,15 @@ +from logging.handlers import BufferingHandler + + +class FTBufferingHandler(BufferingHandler): + def flush(self): + """ + Override Flush behaviour - we keep half of the configured capacity + otherwise, we have moments with "empty" logs. + """ + self.acquire() + try: + # Keep half of the records in buffer. + self.buffer = self.buffer[-int(self.capacity / 2):] + finally: + self.release() diff --git a/freqtrade/loggers/std_err_stream_handler.py b/freqtrade/loggers/std_err_stream_handler.py new file mode 100644 index 000000000..487a7c100 --- /dev/null +++ b/freqtrade/loggers/std_err_stream_handler.py @@ -0,0 +1,26 @@ +import sys +from logging import Handler + + +class FTStdErrStreamHandler(Handler): + def flush(self): + """ + Override Flush behaviour - we keep half of the configured capacity + otherwise, we have moments with "empty" logs. + """ + self.acquire() + try: + sys.stderr.flush() + finally: + self.release() + + def emit(self, record): + try: + msg = self.format(record) + # Don't keep a reference to stderr - this can be problematic with progressbars. + sys.stderr.write(msg + '\n') + self.flush() + except RecursionError: + raise + except Exception: + self.handleError(record) From 4c1de4ad56c4d4ade00ee6cb646d24043563f793 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 18:07:28 +0200 Subject: [PATCH 6/9] Update tests --- tests/test_configuration.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index aab868bec..c445b989d 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -23,7 +23,8 @@ from freqtrade.configuration.load_config import (load_config_file, load_file, lo from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.loggers import FTBufferingHandler, _set_loggers, setup_logging, setup_logging_pre +from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, _set_loggers, + setup_logging, setup_logging_pre) from tests.conftest import (CURRENT_TEST_STRATEGY, log_has, log_has_re, patched_configuration_load_config_file) @@ -658,7 +659,7 @@ def test_set_loggers_syslog(): setup_logging(config) assert len(logger.handlers) == 3 assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] - assert [x for x in logger.handlers if type(x) == logging.StreamHandler] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] assert [x for x in logger.handlers if type(x) == FTBufferingHandler] # setting up logging again should NOT cause the loggers to be added a second time. setup_logging(config) @@ -681,7 +682,7 @@ def test_set_loggers_Filehandler(tmpdir): setup_logging(config) assert len(logger.handlers) == 3 assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler] - assert [x for x in logger.handlers if type(x) == logging.StreamHandler] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] assert [x for x in logger.handlers if type(x) == FTBufferingHandler] # setting up logging again should NOT cause the loggers to be added a second time. setup_logging(config) @@ -706,7 +707,7 @@ def test_set_loggers_journald(mocker): setup_logging(config) assert len(logger.handlers) == 3 assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] - assert [x for x in logger.handlers if type(x) == logging.StreamHandler] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] # reset handlers to not break pytest logger.handlers = orig_handlers From 299e78889143bf5b019a10874284bf40c81c9186 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 17:01:07 +0200 Subject: [PATCH 7/9] Dump progressbar2 dependency --- requirements-hyperopt.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 2c7c27d98..f7ee91663 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,4 +6,3 @@ scipy==1.10.1 scikit-learn==1.1.3 scikit-optimize==0.9.0 filelock==3.10.6 -progressbar2==4.2.0 diff --git a/setup.py b/setup.py index 5c95e9316..048dc066d 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,6 @@ hyperopt = [ 'scikit-learn', 'scikit-optimize>=0.7.0', 'filelock', - 'progressbar2', ] freqai = [ From bfd9e35e34f2a0a6b3c0ff8547390be8cb551c0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 18:09:15 +0200 Subject: [PATCH 8/9] Replace hyperopt progressbar with rich progressbar --- freqtrade/optimize/hyperopt.py | 50 ++++++++++------------------------ 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 96c95c4a2..53d85dfd1 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -13,13 +13,13 @@ from math import ceil from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -import progressbar import rapidjson -from colorama import Fore, Style from colorama import init as colorama_init from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects from joblib.externals import cloudpickle from pandas import DataFrame +from rich.progress import (BarColumn, MofNCompleteColumn, Progress, TaskProgressColumn, TextColumn, + TimeElapsedColumn, TimeRemainingColumn) from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config from freqtrade.data.converter import trim_dataframes @@ -44,8 +44,6 @@ with warnings.catch_warnings(): from skopt import Optimizer from skopt.space import Dimension -progressbar.streams.wrap_stderr() -progressbar.streams.wrap_stdout() logger = logging.getLogger(__name__) @@ -520,29 +518,6 @@ class Hyperopt: else: return self.opt.ask(n_points=n_points), [False for _ in range(n_points)] - def get_progressbar_widgets(self): - if self.print_colorized: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - fill_wrap=Fore.GREEN + '{}' + Fore.RESET, - marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] - else: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] - return widgets - def evaluate_result(self, val: Dict[str, Any], current: int, is_random: bool): """ Evaluate results returned from generate_optimizer @@ -602,11 +577,18 @@ class Hyperopt: logger.info(f'Effective number of parallel workers used: {jobs}') # Define progressbar - widgets = self.get_progressbar_widgets() - with progressbar.ProgressBar( - max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, - widgets=widgets + with Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(bar_width=None), + MofNCompleteColumn(), + TaskProgressColumn(), + TimeElapsedColumn(), + "<", + TimeRemainingColumn(), + expand=True, ) as pbar: + task = pbar.add_task("Epochs", total=self.total_epochs) + start = 0 if self.analyze_per_epoch: @@ -616,7 +598,7 @@ class Hyperopt: f_val0 = self.generate_optimizer(asked[0]) self.opt.tell(asked, [f_val0['loss']]) self.evaluate_result(f_val0, 1, is_random[0]) - pbar.update(1) + pbar.update(task, advance=1) start += 1 evals = ceil((self.total_epochs - start) / jobs) @@ -630,14 +612,12 @@ class Hyperopt: f_val = self.run_optimizer_parallel(parallel, asked) self.opt.tell(asked, [v['loss'] for v in f_val]) - # Calculate progressbar outputs for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 + start self.evaluate_result(val, current, is_random[j]) - - pbar.update(current) + pbar.update(task, advance=1) except KeyboardInterrupt: print('User interrupted..') From cf770d496b6608d5a5f96256bc2ec2575eb9d729 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 18:25:50 +0200 Subject: [PATCH 9/9] Improve visual display of progressbar --- freqtrade/optimize/hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 53d85dfd1..ee5599e20 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -582,8 +582,9 @@ class Hyperopt: BarColumn(bar_width=None), MofNCompleteColumn(), TaskProgressColumn(), + "•", TimeElapsedColumn(), - "<", + "•", TimeRemainingColumn(), expand=True, ) as pbar: