diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0ca99ec0a..25fc78f0a 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -165,7 +165,7 @@ to find optimal parameter values for your stategy. ``` usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] - [--timerange TIMERANGE] [-e INT] [--use-mongodb] + [--timerange TIMERANGE] [-e INT] [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] optional arguments: @@ -175,11 +175,8 @@ optional arguments: --realistic-simulation uses max_open_trades from config to simulate real world limitations - --timerange TIMERANGE - specify what timerange of data to use. + --timerange TIMERANGE specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) - --use-mongodb parallelize evaluations with mongodb (requires mongod - in PATH) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate list. Default: all diff --git a/docs/hyperopt.md b/docs/hyperopt.md index a079e34df..2ad94896a 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -9,7 +9,6 @@ parameters with Hyperopt. - [Advanced Hyperopt notions](#advanced-notions) - [Understand the Guards and Triggers](#understand-the-guards-and-triggers) - [Execute Hyperopt](#execute-hyperopt) - - [Hyperopt with MongoDB](#hyperopt-with-mongoDB) - [Understand the hyperopts result](#understand-the-backtesting-result) ## Prepare Hyperopt @@ -194,41 +193,6 @@ Legal values are: - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` -### Hyperopt with MongoDB -Hyperopt with MongoDB, is like Hyperopt under steroids. As you saw by -executing the previous command is the execution takes a long time. -To accelerate it you can use hyperopt with MongoDB. - -To run hyperopt with MongoDb you will need 3 terminals. - -**Terminal 1: Start MongoDB** -```bash -cd -source .env/bin/activate -python3 scripts/start-mongodb.py -``` - -**Terminal 2: Start Hyperopt worker** -```bash -cd -source .env/bin/activate -python3 scripts/start-hyperopt-worker.py -``` - -**Terminal 3: Start Hyperopt with MongoDB** -```bash -cd -source .env/bin/activate -python3 ./freqtrade/main.py -c config.json hyperopt --use-mongodb -``` - -**Re-run an Hyperopt** -To re-run Hyperopt you have to delete the existing MongoDB table. -```bash -cd -rm -rf .hyperopt/mongodb/ -``` - ## Understand the hyperopts result Once Hyperopt is completed you can use the result to adding new buy signal. Given following result from hyperopt: diff --git a/docs/installation.md b/docs/installation.md index 9818529f6..b6688885b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -225,17 +225,7 @@ cd .. rm -rf ./ta-lib* ``` -#### 3. [Optional] Install MongoDB - -Install MongoDB if you plan to optimize your strategy with Hyperopt. - -```bash -sudo apt-get install mongodb-org -``` - -> Complete tutorial from Digital Ocean: [How to Install MongoDB on Ubuntu 16.04](https://www.digitalocean.com/community/tutorials/how-to-install-mongodb-on-ubuntu-16-04). - -#### 4. Install FreqTrade +#### 3. Install FreqTrade Clone the git repository: @@ -243,7 +233,7 @@ Clone the git repository: git clone https://github.com/freqtrade/freqtrade.git ``` -#### 5. Configure `freqtrade` as a `systemd` service +#### 4. Configure `freqtrade` as a `systemd` service From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. @@ -267,19 +257,7 @@ sudo loginctl enable-linger "$USER" brew install python3 git wget ta-lib ``` -#### 2. [Optional] Install MongoDB - -Install MongoDB if you plan to optimize your strategy with Hyperopt. - -```bash -curl -O https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.4.10.tgz -tar -zxvf mongodb-osx-ssl-x86_64-3.4.10.tgz -mkdir -p /env/mongodb -cp -R -n mongodb-osx-x86_64-3.4.10/ /env/mongodb -export PATH=/env/mongodb/bin:$PATH -``` - -#### 3. Install FreqTrade +#### 2. Install FreqTrade Clone the git repository: diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 331bb73a0..b392fb53e 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -203,12 +203,6 @@ class Arguments(object): type=int, metavar='INT', ) - parser.add_argument( - '--use-mongodb', - help='parallelize evaluations with mongodb (requires mongod in PATH)', - dest='mongodb', - action='store_true', - ) parser.add_argument( '-s', '--spaces', help='Specify which parameters to hyperopt. Space separate list. \ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 1f14df560..7c3a5eb4b 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -188,11 +188,6 @@ class Configuration(object): logger.info('Parameter --epochs detected ...') logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) - # If --mongodb is used we add it to the configuration - if 'mongodb' in self.args and self.args.mongodb: - config.update({'mongodb': self.args.mongodb}) - logger.info('Parameter --use-mongodb detected ...') - # If --spaces is used we add it to the configuration if 'spaces' in self.args and self.args.spaces: config.update({'spaces': self.args.spaces}) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5acd05766..978d1fe2c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -19,7 +19,6 @@ from typing import Dict, Any, Callable, Optional import numpy import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe -from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib @@ -507,32 +506,20 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - if self.config.get('mongodb'): - logger.info('Using mongodb ...') + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries logger.info( - 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' + 'Continuing with trials. Current: %d, Total: %d', + self.current_tries, + self.total_tries ) - db_name = 'freqtrade_hyperopt' - self.trials = MongoTrials( - arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), - exp_key='exp1' - ) - else: - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, self.signal_handler) - # read trials file if we have one - if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: - self.trials = self.read_trials() - - self.current_tries = len(self.trials.results) - self.total_tries += self.current_tries - logger.info( - 'Continuing with trials. Current: %d, Total: %d', - self.current_tries, - self.total_tries - ) - try: best_parameters = fmin( fn=self.generate_optimizer, @@ -588,7 +575,6 @@ def start(args: Namespace) -> None: """ # Remove noisy log messages - logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) # Initialize configuration diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 4ef5762e1..ce7cd77c8 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -247,7 +247,6 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'epochs': 1}) - conf.update({'mongodb': False}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) @@ -341,7 +340,6 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'epochs': 1}) - conf.update({'mongodb': False}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) @@ -353,35 +351,6 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: mock_fmin.assert_called_once() -def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None: - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - mock_mongotrials = mocker.patch( - 'freqtrade.optimize.hyperopt.MongoTrials', - return_value=create_trials(mocker) - ) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'mongodb': True}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - - hyperopt = Hyperopt(conf) - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - mock_mongotrials.assert_called_once() - mock_fmin.assert_called_once() - - -# test log_trials_result -# test buy_strategy_generator def populate_buy_trend -# test optimizer if 'ro_t1' in params - def test_format_results(init_hyperopt): """ Test Hyperopt.format_results() diff --git a/freqtrade/tests/optimize/test_hyperopt_config.py b/freqtrade/tests/optimize/test_hyperopt_config.py deleted file mode 100644 index aa9424826..000000000 --- a/freqtrade/tests/optimize/test_hyperopt_config.py +++ /dev/null @@ -1,16 +0,0 @@ -# pragma pylint: disable=missing-docstring,W0212 - -from user_data.hyperopt_conf import hyperopt_optimize_conf - - -def test_hyperopt_optimize_conf(): - hyperopt_conf = hyperopt_optimize_conf() - - assert "max_open_trades" in hyperopt_conf - assert "stake_currency" in hyperopt_conf - assert "stake_amount" in hyperopt_conf - assert "minimal_roi" in hyperopt_conf - assert "stoploss" in hyperopt_conf - assert "bid_strategy" in hyperopt_conf - assert "exchange" in hyperopt_conf - assert "pair_whitelist" in hyperopt_conf['exchange'] diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index caaddbf25..212df2e96 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -310,7 +310,6 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: arglist = [ 'hyperopt', '--epochs', '10', - '--use-mongodb', '--spaces', 'all', ] @@ -324,10 +323,6 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert log_has('Parameter --epochs detected ...', caplog.record_tuples) assert log_has('Will run Hyperopt with for 10 epochs ...', caplog.record_tuples) - assert 'mongodb' in config - assert config['mongodb'] is True - assert log_has('Parameter --use-mongodb detected ...', caplog.record_tuples) - assert 'spaces' in config assert config['spaces'] == ['all'] assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py deleted file mode 100755 index 8b0ae6326..000000000 --- a/scripts/start-hyperopt-worker.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -import multiprocessing -import os -import subprocess - -PROC_COUNT = multiprocessing.cpu_count() - 1 -DB_NAME = 'freqtrade_hyperopt' -WORK_DIR = os.path.join( - os.path.sep, - os.path.abspath(os.path.dirname(__file__)), - '..', '.hyperopt', 'worker' -) -if not os.path.exists(WORK_DIR): - os.makedirs(WORK_DIR) - -# Spawn workers -command = [ - 'hyperopt-mongo-worker', - '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), - '--poll-interval=0.1', - '--workdir={}'.format(WORK_DIR), -] -processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] - -# Join all workers -for proc in processes: - proc.wait() diff --git a/scripts/start-mongodb.py b/scripts/start-mongodb.py deleted file mode 100755 index 910ee9233..000000000 --- a/scripts/start-mongodb.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 - -import os -import subprocess - - -DB_PATH = os.path.join( - os.path.sep, - os.path.abspath(os.path.dirname(__file__)), - '..', '.hyperopt', 'mongodb' -) -if not os.path.exists(DB_PATH): - os.makedirs(DB_PATH) - -subprocess.Popen([ - 'mongod', - '--bind_ip=127.0.0.1', - '--port=1234', - '--nohttpinterface', - '--dbpath={}'.format(DB_PATH), -]).wait()