Merge pull request #3021 from Fredrik81/print-csv

Hyperopt: Add export CSV-file option

docs/utils.md

@@ -429,6 +429,7 @@ usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [--min-total-profit FLOAT]
                                [--max-total-profit FLOAT] [--no-color]
                                [--print-json] [--no-details]
+                               [--export-csv FILE]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -450,6 +451,8 @@ optional arguments:
                         useful if you are redirecting output to a file.
   --print-json          Print best result detailization in JSON format.
   --no-details          Do not print best epoch details.
+  --export-csv FILE     Export to CSV-File. This will disable table print.
+                        Example: --export-csv hyperopt.csv
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
@@ -458,9 +461,10 @@ Common arguments:
                         details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`).
+                        Specify configuration file (default:
-                        Multiple --config options may be used. Can be set to
+                        `userdir/config.json` or `config.json` whichever
-                        `-` to read config from stdin.
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
   -d PATH, --datadir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH

freqtrade/commands/arguments.py

@@ -69,7 +69,8 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
                       "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
                       "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit",
                       "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit",
-                      "print_colorized", "print_json", "hyperopt_list_no_details"]
+                      "print_colorized", "print_json", "hyperopt_list_no_details",
+                      "export_csv"]
 
 ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
                       "print_json", "hyperopt_show_no_header"]

freqtrade/commands/cli_options.py

@@ -221,6 +221,13 @@ AVAILABLE_CLI_OPTIONS = {
         action='store_true',
         default=False,
     ),
+    "export_csv": Arg(
+        '--export-csv',
+        help='Export to CSV-File.'
+        ' This will disable table print.'
+        ' Example: --export-csv hyperopt.csv',
+        metavar='FILE',
+    ),
     "hyperopt_jobs": Arg(
         '-j', '--job-workers',
         help='The number of concurrently running jobs for hyperoptimization '

freqtrade/commands/hyperopt_commands.py

@@ -21,6 +21,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
 
     print_colorized = config.get('print_colorized', False)
     print_json = config.get('print_json', False)
+    export_csv = config.get('export_csv', None)
     no_details = config.get('hyperopt_list_no_details', False)
     no_header = False
 
@@ -49,17 +50,23 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     if print_colorized:
         colorama_init(autoreset=True)
 
-    try:
+    if not export_csv:
-        Hyperopt.print_result_table(config, trials, total_epochs,
+        try:
-                                    not filteroptions['only_best'], print_colorized, 0)
+            Hyperopt.print_result_table(config, trials, total_epochs,
-    except KeyboardInterrupt:
+                                        not filteroptions['only_best'], print_colorized, 0)
-        print('User interrupted..')
+        except KeyboardInterrupt:
+            print('User interrupted..')
 
     if trials and not no_details:
         sorted_trials = sorted(trials, key=itemgetter('loss'))
         results = sorted_trials[0]
         Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
 
+    if trials and export_csv:
+        Hyperopt.export_csv_file(
+            config, trials, total_epochs, not filteroptions['only_best'], export_csv
+        )
+
 
 def start_hyperopt_show(args: Dict[str, Any]) -> None:
     """

freqtrade/configuration/configuration.py

@@ -282,6 +282,9 @@ class Configuration:
         self._args_to_config(config, argname='print_json',
                              logstring='Parameter --print-json detected ...')
 
+        self._args_to_config(config, argname='export_csv',
+                             logstring='Parameter --export-csv detected: {}')
+
         self._args_to_config(config, argname='hyperopt_jobs',
                              logstring='Parameter -j/--job-workers detected: {}')
 

freqtrade/optimize/hyperopt.py

@@ -23,6 +23,8 @@ from joblib import (Parallel, cpu_count, delayed, dump, load,
                     wrap_non_picklable_objects)
 from pandas import DataFrame, json_normalize, isna
 import tabulate
+from os import path
+import io
 
 from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.history import get_timerange
@@ -330,10 +332,10 @@ class Hyperopt:
             lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
         )
         trials['Avg profit'] = trials['Avg profit'].apply(
-            lambda x: ('{:,.2f}%'.format(x)).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
+            lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
         )
         trials['Avg duration'] = trials['Avg duration'].apply(
-            lambda x: ('{:,.1f} m'.format(x)).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
+            lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
         )
         trials['Objective'] = trials['Objective'].apply(
             lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
@@ -381,6 +383,62 @@ class Hyperopt:
             )
         print(table)
 
+    @staticmethod
+    def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool,
+                        csv_file: str) -> None:
+        """
+        Log result to csv-file
+        """
+        if not results:
+            return
+
+        # Verification for overwrite
+        if path.isfile(csv_file):
+            logger.error("CSV-File already exists!")
+            return
+
+        try:
+            io.open(csv_file, 'w+').close()
+        except IOError:
+            logger.error("Filed to create CSV-File!")
+            return
+
+        trials = json_normalize(results, max_level=1)
+        trials['Best'] = ''
+        trials['Stake currency'] = config['stake_currency']
+        trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
+                         'results_metrics.avg_profit', 'results_metrics.total_profit',
+                         'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
+                         'loss', 'is_initial_point', 'is_best']]
+        trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
+                          'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
+        trials['is_profit'] = False
+        trials.loc[trials['is_initial_point'], 'Best'] = '*'
+        trials.loc[trials['is_best'], 'Best'] = 'Best'
+        trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
+        trials['Epoch'] = trials['Epoch'].astype(str)
+        trials['Trades'] = trials['Trades'].astype(str)
+
+        trials['Total profit'] = trials['Total profit'].apply(
+            lambda x: '{:,.8f}'.format(x) if x != 0.0 else ""
+        )
+        trials['Profit'] = trials['Profit'].apply(
+            lambda x: '{:,.2f}'.format(x) if not isna(x) else ""
+        )
+        trials['Avg profit'] = trials['Avg profit'].apply(
+            lambda x: '{:,.2f}%'.format(x) if not isna(x) else ""
+        )
+        trials['Avg duration'] = trials['Avg duration'].apply(
+            lambda x: '{:,.1f} m'.format(x) if not isna(x) else ""
+        )
+        trials['Objective'] = trials['Objective'].apply(
+            lambda x: '{:,.5f}'.format(x) if x != 100000 else ""
+        )
+
+        trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
+        trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8')
+        print("CSV-File created!")
+
     def has_space(self, space: str) -> bool:
         """
         Tell if the space value is contained in the configuration

tests/commands/test_commands.py

@@ -902,6 +902,21 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results):
     assert all(x not in captured.out
                for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12"
                          " 9/12", " 10/12", " 11/12", " 12/12"])
+    args = [
+        "hyperopt-list",
+        "--no-details",
+        "--export-csv", "test_file.csv"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_hyperopt_list(pargs)
+    captured = capsys.readouterr()
+    assert all(x in captured.out
+               for x in ["CSV-File created!"])
+    f = Path("test_file.csv")
+    assert 'Best,1,2,-1.25%,-0.00125625,,-2.51,"3,930.0 m",0.43662' in f.read_text()
+    assert f.is_file()
+    f.unlink()
 
 
 def test_hyperopt_show(mocker, capsys, hyperopt_results):