Refactor Hyperopt-list and hyperopt-show to reduce some duplicate code

freqtrade/commands/hyperopt_commands.py

@@ -1,6 +1,6 @@
 import logging
 from operator import itemgetter
-from typing import Any, Dict, List
+from typing import Any, Dict
 
 from colorama import init as colorama_init
 
@@ -28,30 +28,12 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     no_details = config.get('hyperopt_list_no_details', False)
     no_header = False
 
-    filteroptions = {
-        'only_best': config.get('hyperopt_list_best', False),
-        'only_profitable': config.get('hyperopt_list_profitable', False),
-        'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
-        'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
-        'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
-        'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
-        'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
-        'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
-        'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
-        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
-        'filter_max_objective': config.get('hyperopt_list_max_objective', None),
-    }
-
     results_file = get_latest_hyperopt_file(
         config['user_data_dir'] / 'hyperopt_results',
         config.get('hyperoptexportfilename'))
 
     # Previous evaluations
-    epochs = HyperoptTools.load_previous_results(results_file)
+    epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
-    total_epochs = len(epochs)
-
-    epochs = hyperopt_filter_epochs(epochs, filteroptions)
 
     if print_colorized:
         colorama_init(autoreset=True)
@@ -59,7 +41,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     if not export_csv:
         try:
             print(HyperoptTools.get_result_table(config, epochs, total_epochs,
-                                                 not filteroptions['only_best'],
+                                                 not config.get('hyperopt_list_best', False),
                                                  print_colorized, 0))
         except KeyboardInterrupt:
             print('User interrupted..')
@@ -71,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
 
     if epochs and export_csv:
         HyperoptTools.export_csv_file(
-            config, epochs, total_epochs, not filteroptions['only_best'], export_csv
+            config, epochs, total_epochs, not config.get('hyperopt_list_best', False), export_csv
         )
 
 
@@ -91,26 +73,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
     n = config.get('hyperopt_show_index', -1)
 
-    filteroptions = {
-        'only_best': config.get('hyperopt_list_best', False),
-        'only_profitable': config.get('hyperopt_list_profitable', False),
-        'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
-        'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
-        'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
-        'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
-        'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
-        'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
-        'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
-        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
-        'filter_max_objective': config.get('hyperopt_list_max_objective', None)
-    }
-
     # Previous evaluations
-    epochs = HyperoptTools.load_previous_results(results_file)
+    epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
-    total_epochs = len(epochs)
 
-    epochs = hyperopt_filter_epochs(epochs, filteroptions)
     filtered_epochs = len(epochs)
 
     if n > filtered_epochs:
@@ -137,138 +102,3 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
         HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
                                          header_str="Epoch details")
-
-
-def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
-    """
-    Filter our items from the list of hyperopt results
-    TODO: after 2021.5 remove all "legacy" mode queries.
-    """
-    if filteroptions['only_best']:
-        epochs = [x for x in epochs if x['is_best']]
-    if filteroptions['only_profitable']:
-        epochs = [x for x in epochs if x['results_metrics'].get(
-            'profit', x['results_metrics'].get('profit_total', 0)) > 0]
-
-    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
-
-    logger.info(f"{len(epochs)} " +
-                ("best " if filteroptions['only_best'] else "") +
-                ("profitable " if filteroptions['only_profitable'] else "") +
-                "epochs found.")
-    return epochs
-
-
-def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
-    """
-    Filter epochs with trade-counts > trades
-    """
-    return [
-        x for x in epochs
-        if x['results_metrics'].get(
-            'trade_count', x['results_metrics'].get('total_trades', 0)
-        ) > trade_count
-    ]
-
-
-def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_trades'] > 0:
-        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
-
-    if filteroptions['filter_max_trades'] > 0:
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'trade_count', x['results_metrics'].get('total_trades')
-            ) < filteroptions['filter_max_trades']
-        ]
-    return epochs
-
-
-def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
-
-    def get_duration_value(x):
-        # Duration in minutes ...
-        if 'duration' in x['results_metrics']:
-            return x['results_metrics']['duration']
-        else:
-            # New mode
-            if 'holding_avg_s' in x['results_metrics']:
-                avg = x['results_metrics']['holding_avg_s']
-                return avg // 60
-            raise OperationalException(
-                "Holding-average not available. Please omit the filter on average time, "
-                "or rerun hyperopt with this version")
-
-    if filteroptions['filter_min_avg_time'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if get_duration_value(x) > filteroptions['filter_min_avg_time']
-        ]
-    if filteroptions['filter_max_avg_time'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if get_duration_value(x) < filteroptions['filter_max_avg_time']
-        ]
-
-    return epochs
-
-
-def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_avg_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
-            ) > filteroptions['filter_min_avg_profit']
-        ]
-    if filteroptions['filter_max_avg_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
-            ) < filteroptions['filter_max_avg_profit']
-        ]
-    if filteroptions['filter_min_total_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'profit', x['results_metrics'].get('profit_total_abs', 0)
-            ) > filteroptions['filter_min_total_profit']
-        ]
-    if filteroptions['filter_max_total_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'profit', x['results_metrics'].get('profit_total_abs', 0)
-            ) < filteroptions['filter_max_total_profit']
-        ]
-    return epochs
-
-
-def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_objective'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-
-        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
-    if filteroptions['filter_max_objective'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-
-        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
-
-    return epochs

freqtrade/optimize/hyperopt_epoch_filters.py

@@ -0,0 +1,142 @@
+import logging
+from typing import List
+
+from freqtrade.exceptions import OperationalException
+
+
+logger = logging.getLogger(__name__)
+
+
+def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
+    """
+    Filter our items from the list of hyperopt results
+    TODO: after 2021.5 remove all "legacy" mode queries.
+    """
+    if filteroptions['only_best']:
+        epochs = [x for x in epochs if x['is_best']]
+    if filteroptions['only_profitable']:
+        epochs = [x for x in epochs if x['results_metrics'].get(
+            'profit', x['results_metrics'].get('profit_total', 0)) > 0]
+
+    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
+
+    logger.info(f"{len(epochs)} " +
+                ("best " if filteroptions['only_best'] else "") +
+                ("profitable " if filteroptions['only_profitable'] else "") +
+                "epochs found.")
+    return epochs
+
+
+def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
+    """
+    Filter epochs with trade-counts > trades
+    """
+    return [
+        x for x in epochs
+        if x['results_metrics'].get(
+            'trade_count', x['results_metrics'].get('total_trades', 0)
+        ) > trade_count
+    ]
+
+
+def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_trades'] > 0:
+        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
+
+    if filteroptions['filter_max_trades'] > 0:
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get(
+                'trade_count', x['results_metrics'].get('total_trades')
+            ) < filteroptions['filter_max_trades']
+        ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
+
+    def get_duration_value(x):
+        # Duration in minutes ...
+        if 'duration' in x['results_metrics']:
+            return x['results_metrics']['duration']
+        else:
+            # New mode
+            if 'holding_avg_s' in x['results_metrics']:
+                avg = x['results_metrics']['holding_avg_s']
+                return avg // 60
+            raise OperationalException(
+                "Holding-average not available. Please omit the filter on average time, "
+                "or rerun hyperopt with this version")
+
+    if filteroptions['filter_min_avg_time'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if get_duration_value(x) > filteroptions['filter_min_avg_time']
+        ]
+    if filteroptions['filter_max_avg_time'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if get_duration_value(x) < filteroptions['filter_max_avg_time']
+        ]
+
+    return epochs
+
+
+def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_avg_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get(
+                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
+            ) > filteroptions['filter_min_avg_profit']
+        ]
+    if filteroptions['filter_max_avg_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get(
+                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
+            ) < filteroptions['filter_max_avg_profit']
+        ]
+    if filteroptions['filter_min_total_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get(
+                'profit', x['results_metrics'].get('profit_total_abs', 0)
+            ) > filteroptions['filter_min_total_profit']
+        ]
+    if filteroptions['filter_max_total_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get(
+                'profit', x['results_metrics'].get('profit_total_abs', 0)
+            ) < filteroptions['filter_max_total_profit']
+        ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_objective'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+
+        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
+    if filteroptions['filter_max_objective'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+
+        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
+
+    return epochs

freqtrade/optimize/hyperopt_tools.py

@@ -4,7 +4,7 @@ import logging
 from copy import deepcopy
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 import numpy as np
 import rapidjson
@@ -15,6 +15,7 @@ from pandas import isna, json_normalize
 from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
+from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
 
 
 logger = logging.getLogger(__name__)
@@ -130,6 +131,31 @@ class HyperoptTools():
             logger.info(f"Loaded {len(epochs)} previous evaluations from disk.")
         return epochs
 
+    @staticmethod
+    def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]:
+        filteroptions = {
+            'only_best': config.get('hyperopt_list_best', False),
+            'only_profitable': config.get('hyperopt_list_profitable', False),
+            'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
+            'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
+            'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
+            'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
+            'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
+            'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
+            'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
+            'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
+            'filter_min_objective': config.get('hyperopt_list_min_objective', None),
+            'filter_max_objective': config.get('hyperopt_list_max_objective', None),
+        }
+
+        # Previous evaluations
+        epochs = HyperoptTools.load_previous_results(results_file)
+        total_epochs = len(epochs)
+
+        epochs = hyperopt_filter_epochs(epochs, filteroptions)
+
+        return epochs, total_epochs
+
     @staticmethod
     def show_epoch_details(results, total_epochs: int, print_json: bool,
                            no_header: bool = False, header_str: str = None) -> None: