Merge pull request #1791 from hroff-1902/hyperopt-refresh-pairs

hyperopt: --refresh-pairs-cached added

docs/bot-usage.md

@@ -146,9 +146,11 @@ Backtesting also uses the config specified via `-c/--config`.
 
 ```
 usage: freqtrade backtesting [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                             [--eps] [--dmmp] [-l] [-r]
+                           [--max_open_trades MAX_OPEN_TRADES]
-                             [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
+                           [--stake_amount STAKE_AMOUNT] [-r] [--eps] [--dmmp]
-                             [--export EXPORT] [--export-filename PATH]
+                           [-l]
+                           [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
+                           [--export EXPORT] [--export-filename PATH]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -156,6 +158,14 @@ optional arguments:
                         Specify ticker interval (1m, 5m, 30m, 1h, 1d).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
+  --max_open_trades MAX_OPEN_TRADES
+                        Specify max_open_trades to use.
+  --stake_amount STAKE_AMOUNT
+                        Specify stake_amount.
+  -r, --refresh-pairs-cached
+                        Refresh the pairs files in tests/testdata with the
+                        latest data from the exchange. Use it if you want to
+                        run your optimization commands with up-to-date data.
   --eps, --enable-position-stacking
                         Allow buying the same pair multiple times (position
                         stacking).
@@ -164,10 +174,6 @@ optional arguments:
                         (same as setting `max_open_trades` to a very high
                         number).
   -l, --live            Use live data.
-  -r, --refresh-pairs-cached
-                        Refresh the pairs files in tests/testdata with the
-                        latest data from the exchange. Use it if you want to
-                        run your backtesting with up-to-date data.
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a commaseparated list of strategies to
                         backtest Please note that ticker-interval needs to be
@@ -206,8 +212,11 @@ to find optimal parameter values for your stategy.
 
 ```
 usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                          [--customhyperopt NAME] [--eps] [--dmmp] [-e INT]
+                        [--max_open_trades MAX_OPEN_TRADES]
-                          [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]]
+                        [--stake_amount STAKE_AMOUNT] [-r]
+                        [--customhyperopt NAME] [--eps] [--dmmp] [-e INT]
+                        [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]]
+                        [--print-all]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -215,6 +224,14 @@ optional arguments:
                         Specify ticker interval (1m, 5m, 30m, 1h, 1d).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
+  --max_open_trades MAX_OPEN_TRADES
+                        Specify max_open_trades to use.
+  --stake_amount STAKE_AMOUNT
+                        Specify stake_amount.
+  -r, --refresh-pairs-cached
+                        Refresh the pairs files in tests/testdata with the
+                        latest data from the exchange. Use it if you want to
+                        run your optimization commands with up-to-date data.
   --customhyperopt NAME
                         Specify hyperopt class name (default:
                         DefaultHyperOpts).
@@ -229,7 +246,7 @@ optional arguments:
   -s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...], --spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]
                         Specify which parameters to hyperopt. Space separate
                         list. Default: all.
-
+  --print-all           Print all results, not only the best ones.
 ```
 
 ## Edge commands
@@ -237,8 +254,10 @@ optional arguments:
 To know your trade expectacny and winrate against historical data, you can use Edge.
 
 ```
-usage: freqtrade edge [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE] [-r]
+usage: freqtrade edge [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                      [--stoplosses STOPLOSS_RANGE]
+                    [--max_open_trades MAX_OPEN_TRADES]
+                    [--stake_amount STAKE_AMOUNT] [-r]
+                    [--stoplosses STOPLOSS_RANGE]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -246,10 +265,14 @@ optional arguments:
                         Specify ticker interval (1m, 5m, 30m, 1h, 1d).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
+  --max_open_trades MAX_OPEN_TRADES
+                        Specify max_open_trades to use.
+  --stake_amount STAKE_AMOUNT
+                        Specify stake_amount.
   -r, --refresh-pairs-cached
                         Refresh the pairs files in tests/testdata with the
                         latest data from the exchange. Use it if you want to
-                        run your edge with up-to-date data.
+                        run your optimization commands with up-to-date data.
   --stoplosses STOPLOSS_RANGE
                         Defines a range of stoploss against which edge will
                         assess the strategy the format is "min,max,step"

freqtrade/arguments.py

@@ -141,10 +141,53 @@ class Arguments(object):
             dest='sd_notify',
         )
 
+    @staticmethod
+    def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:
+        """
+        Parses given common arguments for Backtesting, Edge and Hyperopt modules.
+        :param parser:
+        :return:
+        """
+        parser.add_argument(
+            '-i', '--ticker-interval',
+            help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).',
+            dest='ticker_interval',
+            type=str,
+        )
+        parser.add_argument(
+            '--timerange',
+            help='Specify what timerange of data to use.',
+            default=None,
+            type=str,
+            dest='timerange',
+        )
+        parser.add_argument(
+            '--max_open_trades',
+            help='Specify max_open_trades to use.',
+            default=None,
+            type=int,
+            dest='max_open_trades',
+        )
+        parser.add_argument(
+            '--stake_amount',
+            help='Specify stake_amount.',
+            default=None,
+            type=float,
+            dest='stake_amount',
+        )
+        parser.add_argument(
+            '-r', '--refresh-pairs-cached',
+            help='Refresh the pairs files in tests/testdata with the latest data from the '
+                 'exchange. Use it if you want to run your optimization commands with '
+                 'up-to-date data.',
+            action='store_true',
+            dest='refresh_pairs',
+        )
+
     @staticmethod
     def backtesting_options(parser: argparse.ArgumentParser) -> None:
         """
-        Parses given arguments for Backtesting scripts.
+        Parses given arguments for Backtesting module.
         """
         parser.add_argument(
             '--eps', '--enable-position-stacking',
@@ -167,13 +210,6 @@ class Arguments(object):
             action='store_true',
             dest='live',
         )
-        parser.add_argument(
-            '-r', '--refresh-pairs-cached',
-            help='Refresh the pairs files in tests/testdata with the latest data from the '
-                 'exchange. Use it if you want to run your backtesting with up-to-date data.',
-            action='store_true',
-            dest='refresh_pairs',
-        )
         parser.add_argument(
             '--strategy-list',
             help='Provide a commaseparated list of strategies to backtest '
@@ -207,15 +243,8 @@ class Arguments(object):
     @staticmethod
     def edge_options(parser: argparse.ArgumentParser) -> None:
         """
-        Parses given arguments for Backtesting scripts.
+        Parses given arguments for Edge module.
         """
-        parser.add_argument(
-            '-r', '--refresh-pairs-cached',
-            help='Refresh the pairs files in tests/testdata with the latest data from the '
-                 'exchange. Use it if you want to run your edge with up-to-date data.',
-            action='store_true',
-            dest='refresh_pairs',
-        )
         parser.add_argument(
             '--stoplosses',
             help='Defines a range of stoploss against which edge will assess the strategy '
@@ -225,48 +254,10 @@ class Arguments(object):
             dest='stoploss_range',
         )
 
-    @staticmethod
-    def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:
-        """
-        Parses given common arguments for Backtesting and Hyperopt scripts.
-        :param parser:
-        :return:
-        """
-        parser.add_argument(
-            '-i', '--ticker-interval',
-            help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).',
-            dest='ticker_interval',
-            type=str,
-        )
-
-        parser.add_argument(
-            '--timerange',
-            help='Specify what timerange of data to use.',
-            default=None,
-            type=str,
-            dest='timerange',
-        )
-
-        parser.add_argument(
-            '--max_open_trades',
-            help='Specify max_open_trades to use.',
-            default=None,
-            type=int,
-            dest='max_open_trades',
-        )
-
-        parser.add_argument(
-            '--stake_amount',
-            help='Specify stake_amount.',
-            default=None,
-            type=float,
-            dest='stake_amount',
-        )
-
     @staticmethod
     def hyperopt_options(parser: argparse.ArgumentParser) -> None:
         """
-        Parses given arguments for Hyperopt scripts.
+        Parses given arguments for Hyperopt module.
         """
         parser.add_argument(
             '--customhyperopt',

freqtrade/configuration.py

@@ -324,6 +324,11 @@ class Configuration(object):
             config.update({'print_all': self.args.print_all})
             logger.info('Parameter --print-all detected: %s', config.get('print_all'))
 
+        # If -r/--refresh-pairs-cached is used we add it to the configuration
+        if 'refresh_pairs' in self.args and self.args.refresh_pairs:
+            config.update({'refresh_pairs': True})
+            logger.info('Parameter -r/--refresh-pairs-cached detected ...')
+
         return config
 
     def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]:

freqtrade/data/history.py

@@ -116,7 +116,8 @@ def load_pair_history(pair: str,
         return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing)
     else:
         logger.warning('No data for pair: "%s", Interval: %s. '
-                       'Use --refresh-pairs-cached to download the data',
+                       'Use --refresh-pairs-cached option or download_backtest_data.py '
+                       'script to download the data',
                        pair, ticker_interval)
         return None
 

freqtrade/optimize/backtesting.py

@@ -516,6 +516,7 @@ def start(args: Namespace) -> None:
     """
     # Initialize configuration
     config = setup_configuration(args)
+
     logger.info('Starting freqtrade in Backtesting mode')
 
     # Initialize backtesting object

freqtrade/optimize/hyperopt.py

@@ -20,6 +20,7 @@ from pandas import DataFrame
 from skopt import Optimizer
 from skopt.space import Dimension
 
+from freqtrade import DependencyException
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.data.history import load_data
@@ -258,6 +259,8 @@ class Hyperopt(Backtesting):
             datadir=Path(self.config['datadir']) if self.config.get('datadir') else None,
             pairs=self.config['exchange']['pair_whitelist'],
             ticker_interval=self.ticker_interval,
+            refresh_pairs=self.config.get('refresh_pairs', False),
+            exchange=self.exchange,
             timerange=timerange
         )
 
@@ -265,7 +268,10 @@ class Hyperopt(Backtesting):
             self.strategy.advise_indicators = \
                 self.custom_hyperopt.populate_indicators  # type: ignore
         dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE)
+
+        # We don't need exchange instance anymore while running hyperopt
         self.exchange = None  # type: ignore
+
         self.load_previous_results()
 
         cpus = multiprocessing.cpu_count()
@@ -295,22 +301,16 @@ class Hyperopt(Backtesting):
         self.log_trials_result()
 
 
-def start(args: Namespace) -> None:
+def setup_configuration(args: Namespace) -> Dict[str, Any]:
     """
-    Start Backtesting script
+    Prepare the configuration for the Hyperopt module
     :param args: Cli args from Arguments()
-    :return: None
+    :return: Configuration
     """
-
-    # Remove noisy log messages
-    logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
-
-    # Initialize configuration
-    # Monkey patch the configuration with hyperopt_conf.py
     configuration = Configuration(args, RunMode.HYPEROPT)
-    logger.info('Starting freqtrade in Hyperopt mode')
     config = configuration.load_config()
 
+    # Ensure we do not use Exchange credentials
     config['exchange']['key'] = ''
     config['exchange']['secret'] = ''
 
@@ -320,7 +320,25 @@ def start(args: Namespace) -> None:
             "Read the documentation at "
             "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md "
             "to understand how to configure hyperopt.")
-        raise ValueError("--strategy configured but not supported for hyperopt")
+        raise DependencyException("--strategy configured but not supported for hyperopt")
+
+    return config
+
+
+def start(args: Namespace) -> None:
+    """
+    Start Backtesting script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    # Initialize configuration
+    config = setup_configuration(args)
+
+    logger.info('Starting freqtrade in Hyperopt mode')
+
+    # Remove noisy log messages
+    logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
+
     # Initialize backtesting object
     hyperopt = Hyperopt(config)
     hyperopt.start()

freqtrade/tests/data/test_history.py

@@ -68,7 +68,10 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None:
     assert ld is None
     assert log_has(
         'No data for pair: "UNITTEST/BTC", Interval: 7m. '
-        'Use --refresh-pairs-cached to download the data', caplog.record_tuples)
+        'Use --refresh-pairs-cached option or download_backtest_data.py '
+        'script to download the data',
+        caplog.record_tuples
+    )
 
 
 def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
@@ -96,9 +99,12 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
                               refresh_pairs=False,
                               pair='MEME/BTC')
     assert os.path.isfile(file) is False
-    assert log_has('No data for pair: "MEME/BTC", Interval: 1m. '
+    assert log_has(
-                   'Use --refresh-pairs-cached to download the data',
+        'No data for pair: "MEME/BTC", Interval: 1m. '
-                   caplog.record_tuples)
+        'Use --refresh-pairs-cached option or download_backtest_data.py '
+        'script to download the data',
+        caplog.record_tuples
+    )
 
     # download a new pair if refresh_pairs is set
     history.load_pair_history(datadir=None,

freqtrade/tests/optimize/test_backtesting.py

@@ -260,6 +260,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
 
     assert 'refresh_pairs' in config
     assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
+
     assert 'timerange' in config
     assert log_has(
         'Parameter --timerange detected: {} ...'.format(config['timerange']),

freqtrade/tests/optimize/test_hyperopt.py

@@ -1,16 +1,19 @@
 # pragma pylint: disable=missing-docstring,W0212,C0103
 from datetime import datetime
+import json
 import os
 from unittest.mock import MagicMock
 
 import pandas as pd
 import pytest
 
+from freqtrade import DependencyException
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.history import load_tickerdata_file
-from freqtrade.optimize.hyperopt import Hyperopt, start
+from freqtrade.optimize.hyperopt import Hyperopt, start, setup_configuration
 from freqtrade.optimize.default_hyperopt import DefaultHyperOpts
 from freqtrade.resolvers import StrategyResolver, HyperOptResolver
+from freqtrade.state import RunMode
 from freqtrade.tests.conftest import log_has, patch_exchange
 from freqtrade.tests.optimize.test_backtesting import get_args
 
@@ -39,6 +42,112 @@ def create_trials(mocker, hyperopt) -> None:
     return [{'loss': 1, 'result': 'foo', 'params': {}}]
 
 
+def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None:
+    mocker.patch('freqtrade.configuration.open', mocker.mock_open(
+        read_data=json.dumps(default_conf)
+    ))
+
+    args = [
+        '--config', 'config.json',
+        'hyperopt'
+    ]
+
+    config = setup_configuration(get_args(args))
+    assert 'max_open_trades' in config
+    assert 'stake_currency' in config
+    assert 'stake_amount' in config
+    assert 'exchange' in config
+    assert 'pair_whitelist' in config['exchange']
+    assert 'datadir' in config
+    assert log_has(
+        'Using data folder: {} ...'.format(config['datadir']),
+        caplog.record_tuples
+    )
+    assert 'ticker_interval' in config
+    assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
+
+    assert 'live' not in config
+    assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples)
+
+    assert 'position_stacking' not in config
+    assert not log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples)
+
+    assert 'refresh_pairs' not in config
+    assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
+
+    assert 'timerange' not in config
+    assert 'runmode' in config
+    assert config['runmode'] == RunMode.HYPEROPT
+
+
+def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
+    mocker.patch('freqtrade.configuration.open', mocker.mock_open(
+        read_data=json.dumps(default_conf)
+    ))
+    mocker.patch('freqtrade.configuration.Configuration._create_datadir', lambda s, c, x: x)
+
+    args = [
+        '--config', 'config.json',
+        '--datadir', '/foo/bar',
+        'hyperopt',
+        '--ticker-interval', '1m',
+        '--timerange', ':100',
+        '--refresh-pairs-cached',
+        '--enable-position-stacking',
+        '--disable-max-market-positions',
+        '--epochs', '1000',
+        '--spaces', 'all',
+        '--print-all'
+    ]
+
+    config = setup_configuration(get_args(args))
+    assert 'max_open_trades' in config
+    assert 'stake_currency' in config
+    assert 'stake_amount' in config
+    assert 'exchange' in config
+    assert 'pair_whitelist' in config['exchange']
+    assert 'datadir' in config
+    assert config['runmode'] == RunMode.HYPEROPT
+
+    assert log_has(
+        'Using data folder: {} ...'.format(config['datadir']),
+        caplog.record_tuples
+    )
+    assert 'ticker_interval' in config
+    assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
+    assert log_has(
+        'Using ticker_interval: 1m ...',
+        caplog.record_tuples
+    )
+
+    assert 'position_stacking' in config
+    assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples)
+
+    assert 'use_max_market_positions' in config
+    assert log_has('Parameter --disable-max-market-positions detected ...', caplog.record_tuples)
+    assert log_has('max_open_trades set to unlimited ...', caplog.record_tuples)
+
+    assert 'refresh_pairs' in config
+    assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
+
+    assert 'timerange' in config
+    assert log_has(
+        'Parameter --timerange detected: {} ...'.format(config['timerange']),
+        caplog.record_tuples
+    )
+
+    assert 'epochs' in config
+    assert log_has('Parameter --epochs detected ...', caplog.record_tuples)
+
+    assert 'spaces' in config
+    assert log_has(
+        'Parameter -s/--spaces detected: {}'.format(config['spaces']),
+        caplog.record_tuples
+    )
+    assert 'print_all' in config
+    assert log_has('Parameter --print-all detected: True', caplog.record_tuples)
+
+
 def test_hyperoptresolver(mocker, default_conf, caplog) -> None:
 
     mocker.patch(
@@ -72,7 +181,6 @@ def test_start(mocker, default_conf, caplog) -> None:
 
     args = [
         '--config', 'config.json',
-        '--strategy', 'DefaultStrategy',
         'hyperopt',
         '--epochs', '5'
     ]
@@ -107,7 +215,7 @@ def test_start_failure(mocker, default_conf, caplog) -> None:
     ]
     args = get_args(args)
     StrategyResolver({'strategy': 'DefaultStrategy'})
-    with pytest.raises(ValueError):
+    with pytest.raises(DependencyException):
         start(args)
     assert log_has(
         "Please don't use --strategy for hyperopt.",