From 3b37f77a4dd61bdcd87fb59ed3e51fc1dc8c73e5 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Fri, 24 Nov 2017 23:58:35 +0100
Subject: [PATCH 01/12] move backtesting to freqtrade.optimize.backtesting

---
 freqtrade/analyze.py                          | 19 +++----
 freqtrade/misc.py                             | 24 ++-------
 freqtrade/optimize/__init__.py                |  1 +
 .../backtesting.py}                           | 49 ++++++++-----------
 freqtrade/tests/__init__.py                   |  7 +--
 freqtrade/tests/test_analyze.py               | 27 +++++++---
 freqtrade/tests/test_hyperopt.py              |  4 +-
 freqtrade/tests/test_misc.py                  | 33 ++-----------
 freqtrade/tests/test_optimize_backtesting.py  | 18 +++++++
 9 files changed, 80 insertions(+), 102 deletions(-)
 create mode 100644 freqtrade/optimize/__init__.py
 rename freqtrade/{tests/test_backtesting.py => optimize/backtesting.py} (81%)
 create mode 100644 freqtrade/tests/test_optimize_backtesting.py

diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py
index ba57d66c5..8edf86340 100644
--- a/freqtrade/analyze.py
+++ b/freqtrade/analyze.py
@@ -4,6 +4,7 @@ Functions to analyze ticker data with indicators and produce buy and sell signal
 import logging
 from datetime import timedelta
 from enum import Enum
+from typing import List, Dict
 
 import arrow
 import talib.abstract as ta
@@ -113,18 +114,13 @@ def populate_sell_trend(dataframe: DataFrame) -> DataFrame:
     return dataframe
 
 
-def analyze_ticker(pair: str) -> DataFrame:
+def analyze_ticker(ticker_history: List[Dict]) -> DataFrame:
     """
-    Get ticker data for given currency pair, push it to a DataFrame and
+    Parses the given ticker history and returns a populated DataFrame
     add several TA indicators and buy signal to it
     :return DataFrame with ticker data and indicator data
     """
-    ticker_hist = get_ticker_history(pair)
-    if not ticker_hist:
-        logger.warning('Empty ticker history for pair %s', pair)
-        return DataFrame()
-
-    dataframe = parse_ticker_dataframe(ticker_hist)
+    dataframe = parse_ticker_dataframe(ticker_history)
     dataframe = populate_indicators(dataframe)
     dataframe = populate_buy_trend(dataframe)
     dataframe = populate_sell_trend(dataframe)
@@ -137,8 +133,13 @@ def get_signal(pair: str, signal: SignalType) -> bool:
     :param pair: pair in format BTC_ANT or BTC-ANT
     :return: True if pair is good for buying, False otherwise
     """
+    ticker_hist = get_ticker_history(pair)
+    if not ticker_hist:
+        logger.warning('Empty ticker history for pair %s', pair)
+        return False
+
     try:
-        dataframe = analyze_ticker(pair)
+        dataframe = analyze_ticker(ticker_hist)
     except ValueError as ex:
         logger.warning('Unable to analyze ticker for pair %s: %s', pair, str(ex))
         return False
diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 97611601b..784e4903e 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -2,7 +2,6 @@ import argparse
 import enum
 import json
 import logging
-import os
 import time
 from typing import Any, Callable, List, Dict
 
@@ -129,9 +128,11 @@ def parse_args(args: List[str]):
 
 def build_subcommands(parser: argparse.ArgumentParser) -> None:
     """ Builds and attaches all subcommands """
+    from freqtrade.optimize import backtesting
+
     subparsers = parser.add_subparsers(dest='subparser')
     backtest = subparsers.add_parser('backtesting', help='backtesting module')
-    backtest.set_defaults(func=start_backtesting)
+    backtest.set_defaults(func=backtesting.start)
     backtest.add_argument(
         '-l', '--live',
         action='store_true',
@@ -154,25 +155,6 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
     )
 
 
-def start_backtesting(args) -> None:
-    """
-    Exports all args as environment variables and starts backtesting via pytest.
-    :param args: arguments namespace
-    :return:
-    """
-    import pytest
-
-    os.environ.update({
-        'BACKTEST': 'true',
-        'BACKTEST_LIVE': 'true' if args.live else '',
-        'BACKTEST_CONFIG': args.config,
-        'BACKTEST_TICKER_INTERVAL': str(args.ticker_interval),
-        'BACKTEST_REALISTIC_SIMULATION': 'true' if args.realistic_simulation else '',
-    })
-    path = os.path.join(os.path.dirname(__file__), 'tests', 'test_backtesting.py')
-    pytest.main(['-s', path])
-
-
 # Required json-schema for user specified config
 CONF_SCHEMA = {
     'type': 'object',
diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py
new file mode 100644
index 000000000..4cffcee10
--- /dev/null
+++ b/freqtrade/optimize/__init__.py
@@ -0,0 +1 @@
+from . import backtesting
diff --git a/freqtrade/tests/test_backtesting.py b/freqtrade/optimize/backtesting.py
similarity index 81%
rename from freqtrade/tests/test_backtesting.py
rename to freqtrade/optimize/backtesting.py
index 36c701426..ea861ee84 100644
--- a/freqtrade/tests/test_backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -2,11 +2,9 @@
 
 
 import logging
-import os
 from typing import Tuple, Dict
 
 import arrow
-import pytest
 from pandas import DataFrame
 from tabulate import tabulate
 
@@ -83,12 +81,12 @@ def generate_text_table(data: Dict[str, Dict], results: DataFrame, stake_currenc
     return tabulate(tabular_data, headers=headers)
 
 
-def backtest(config: Dict, processed, mocker, max_open_trades=0, realistic=True):
+def backtest(config: Dict, processed: Dict[str, DataFrame],
+             max_open_trades: int = 0, realistic: bool = True) -> DataFrame:
     """
     Implements backtesting functionality
     :param config: config to use
     :param processed: a processed dictionary with format {pair, data}
-    :param mocker: mocker instance
     :param max_open_trades: maximum number of concurrent trades (default: 0, disabled)
     :param realistic: do we try to simulate realistic trades? (default: True)
     :return: DataFrame
@@ -96,7 +94,6 @@ def backtest(config: Dict, processed, mocker, max_open_trades=0, realistic=True)
     trades = []
     trade_count_lock = {}
     exchange._API = Bittrex({'key': '', 'secret': ''})
-    mocker.patch.dict('freqtrade.main._CONF', config)
     for pair, pair_data in processed.items():
         pair_data['buy'], pair_data['sell'] = 0, 0
         ticker = populate_sell_trend(populate_buy_trend(pair_data))
@@ -138,38 +135,23 @@ def backtest(config: Dict, processed, mocker, max_open_trades=0, realistic=True)
     return DataFrame.from_records(trades, columns=labels)
 
 
-def get_max_open_trades(config):
-    if not os.environ.get('BACKTEST_REALISTIC_SIMULATION'):
-        return 0
-    print('Using max_open_trades: {} ...'.format(config['max_open_trades']))
-    return config['max_open_trades']
-
-
-@pytest.mark.skipif(not os.environ.get('BACKTEST'), reason="BACKTEST not set")
-def test_backtest(backtest_conf, mocker):
+def start(args):
     print('')
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    # Load configuration file based on env variable
-    conf_path = os.environ.get('BACKTEST_CONFIG')
-    if conf_path:
-        print('Using config: {} ...'.format(conf_path))
-        config = load_config(conf_path)
-    else:
-        config = backtest_conf
+    print('Using config: {} ...'.format(args.config))
+    config = load_config(args.config)
 
-    # Parse ticker interval
-    ticker_interval = int(os.environ.get('BACKTEST_TICKER_INTERVAL') or 5)
-    print('Using ticker_interval: {} ...'.format(ticker_interval))
+    print('Using ticker_interval: {} ...'.format(args.ticker_interval))
 
     data = {}
-    if os.environ.get('BACKTEST_LIVE'):
+    if args.live:
         print('Downloading data for all pairs in whitelist ...')
         for pair in config['exchange']['pair_whitelist']:
-            data[pair] = exchange.get_ticker_history(pair, ticker_interval)
+            data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
         print('Using local backtesting data (ignoring whitelist in given config)...')
-        data = load_backtesting_data(ticker_interval)
+        data = load_backtesting_data(args.ticker_interval)
 
     print('Using stake_currency: {} ...\nUsing stake_amount: {} ...'.format(
         config['stake_currency'], config['stake_amount']
@@ -181,8 +163,17 @@ def test_backtest(backtest_conf, mocker):
         min_date.isoformat(), max_date.isoformat()
     ))
 
+    max_open_trades = 0
+    if args.realistic_simulation:
+        print('Using max_open_trades: {} ...'.format(config['max_open_trades']))
+        max_open_trades = config['max_open_trades']
+
+    from freqtrade import main
+    main._CONF = config
+
     # Execute backtest and print results
-    realistic = os.environ.get('BACKTEST_REALISTIC_SIMULATION')
-    results = backtest(config, preprocess(data), mocker, get_max_open_trades(config), realistic)
+    results = backtest(
+        config, preprocess(data), max_open_trades, args.realistic_simulation
+    )
     print('====================== BACKTESTING REPORT ======================================\n\n')
     print(generate_text_table(data, results, config['stake_currency']))
diff --git a/freqtrade/tests/__init__.py b/freqtrade/tests/__init__.py
index c5cc708c9..ebebe7c98 100644
--- a/freqtrade/tests/__init__.py
+++ b/freqtrade/tests/__init__.py
@@ -1,16 +1,17 @@
 # pragma pylint: disable=missing-docstring
 import json
 import os
+from typing import Optional, List
 
 
-def load_backtesting_data(ticker_interval: int = 5):
+def load_backtesting_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None):
     path = os.path.abspath(os.path.dirname(__file__))
     result = {}
-    pairs = [
+    _pairs = pairs or [
         'BTC_BCC', 'BTC_ETH', 'BTC_DASH', 'BTC_POWR', 'BTC_ETC',
         'BTC_VTC', 'BTC_WAVES', 'BTC_LSK', 'BTC_XLM', 'BTC_OK',
     ]
-    for pair in pairs:
+    for pair in _pairs:
         with open('{abspath}/testdata/{pair}-{ticker_interval}.json'.format(
             abspath=path,
             pair=pair,
diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py
index c62639997..5af612a7e 100644
--- a/freqtrade/tests/test_analyze.py
+++ b/freqtrade/tests/test_analyze.py
@@ -1,5 +1,6 @@
 # pragma pylint: disable=missing-docstring,W0621
 import json
+from unittest.mock import MagicMock
 
 import arrow
 import pytest
@@ -35,20 +36,30 @@ def test_populates_sell_trend(result):
 
 
 def test_returns_latest_buy_signal(mocker):
-    buydf = DataFrame([{'buy': 1, 'date': arrow.utcnow()}])
-    mocker.patch('freqtrade.analyze.analyze_ticker', return_value=buydf)
+    mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock())
+    mocker.patch(
+        'freqtrade.analyze.analyze_ticker',
+        return_value=DataFrame([{'buy': 1, 'date': arrow.utcnow()}])
+    )
     assert get_signal('BTC-ETH', SignalType.BUY)
 
-    buydf = DataFrame([{'buy': 0, 'date': arrow.utcnow()}])
-    mocker.patch('freqtrade.analyze.analyze_ticker', return_value=buydf)
+    mocker.patch(
+        'freqtrade.analyze.analyze_ticker',
+        return_value=DataFrame([{'buy': 0, 'date': arrow.utcnow()}])
+    )
     assert not get_signal('BTC-ETH', SignalType.BUY)
 
 
 def test_returns_latest_sell_signal(mocker):
-    selldf = DataFrame([{'sell': 1, 'date': arrow.utcnow()}])
-    mocker.patch('freqtrade.analyze.analyze_ticker', return_value=selldf)
+    mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock())
+    mocker.patch(
+        'freqtrade.analyze.analyze_ticker',
+        return_value=DataFrame([{'sell': 1, 'date': arrow.utcnow()}])
+    )
     assert get_signal('BTC-ETH', SignalType.SELL)
 
-    selldf = DataFrame([{'sell': 0, 'date': arrow.utcnow()}])
-    mocker.patch('freqtrade.analyze.analyze_ticker', return_value=selldf)
+    mocker.patch(
+        'freqtrade.analyze.analyze_ticker',
+        return_value=DataFrame([{'sell': 0, 'date': arrow.utcnow()}])
+    )
     assert not get_signal('BTC-ETH', SignalType.SELL)
diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py
index ab4b34674..5face5837 100644
--- a/freqtrade/tests/test_hyperopt.py
+++ b/freqtrade/tests/test_hyperopt.py
@@ -11,9 +11,9 @@ from pandas import DataFrame
 
 from freqtrade import exchange
 from freqtrade.exchange import Bittrex
+from freqtrade.optimize.backtesting import backtest, format_results
+from freqtrade.optimize.backtesting import preprocess
 from freqtrade.tests import load_backtesting_data
-from freqtrade.tests.test_backtesting import backtest, format_results
-from freqtrade.tests.test_backtesting import preprocess
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
 logging.disable(logging.DEBUG)  # disable debug logs that slow backtesting a lot
diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py
index a6f61d58b..9748deeea 100644
--- a/freqtrade/tests/test_misc.py
+++ b/freqtrade/tests/test_misc.py
@@ -1,15 +1,13 @@
 # pragma pylint: disable=missing-docstring,C0103
 import json
-import os
 import time
-from argparse import Namespace
 from copy import deepcopy
 from unittest.mock import MagicMock
 
 import pytest
 from jsonschema import ValidationError
 
-from freqtrade.misc import throttle, parse_args, start_backtesting, load_config
+from freqtrade.misc import throttle, parse_args, load_config
 
 
 def test_throttle():
@@ -64,7 +62,7 @@ def test_parse_args_dynamic_whitelist():
 
 
 def test_parse_args_backtesting(mocker):
-    backtesting_mock = mocker.patch('freqtrade.misc.start_backtesting', MagicMock())
+    backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock())
     args = parse_args(['backtesting'])
     assert args is None
     assert backtesting_mock.call_count == 1
@@ -87,7 +85,7 @@ def test_parse_args_backtesting_invalid():
 
 
 def test_parse_args_backtesting_custom(mocker):
-    backtesting_mock = mocker.patch('freqtrade.misc.start_backtesting', MagicMock())
+    backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock())
     args = parse_args(['-c', 'test_conf.json', 'backtesting', '--live', '--ticker-interval', '1'])
     assert args is None
     assert backtesting_mock.call_count == 1
@@ -101,31 +99,6 @@ def test_parse_args_backtesting_custom(mocker):
     assert call_args.ticker_interval == 1
 
 
-def test_start_backtesting(mocker):
-    pytest_mock = mocker.patch('pytest.main', MagicMock())
-    env_mock = mocker.patch('os.environ', {})
-    args = Namespace(
-        config='config.json',
-        live=True,
-        loglevel=20,
-        ticker_interval=1,
-        realistic_simulation=True,
-    )
-    start_backtesting(args)
-    assert env_mock == {
-        'BACKTEST': 'true',
-        'BACKTEST_LIVE': 'true',
-        'BACKTEST_CONFIG': 'config.json',
-        'BACKTEST_TICKER_INTERVAL': '1',
-        'BACKTEST_REALISTIC_SIMULATION': 'true',
-    }
-    assert pytest_mock.call_count == 1
-
-    main_call_args = pytest_mock.call_args[0][0]
-    assert main_call_args[0] == '-s'
-    assert main_call_args[1].endswith(os.path.join('freqtrade', 'tests', 'test_backtesting.py'))
-
-
 def test_load_config(default_conf, mocker):
     file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open(
         read_data=json.dumps(default_conf)
diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py
new file mode 100644
index 000000000..36f4cd144
--- /dev/null
+++ b/freqtrade/tests/test_optimize_backtesting.py
@@ -0,0 +1,18 @@
+# pragma pylint: disable=missing-docstring,W0212
+
+
+from freqtrade import exchange
+from freqtrade.exchange import Bittrex
+from freqtrade.optimize.backtesting import backtest, preprocess
+from freqtrade.tests import load_backtesting_data
+
+
+def test_backtest(backtest_conf, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', backtest_conf)
+    exchange._API = Bittrex({'key': '', 'secret': ''})
+
+    data = load_backtesting_data(ticker_interval=5, pairs=['BTC_ETH'])
+    results = backtest(backtest_conf, preprocess(data), 10, True)
+    num_resutls = len(results)
+    assert num_resutls > 0
+

From 7fa5846c6ba836be4094d96f02566131ed5a070f Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 00:30:39 +0100
Subject: [PATCH 02/12] move hyperopt to freqtrade.optimize.hyperopt

---
 freqtrade/{tests/test_hyperopt.py => optimize/hyperopt.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename freqtrade/{tests/test_hyperopt.py => optimize/hyperopt.py} (100%)

diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/optimize/hyperopt.py
similarity index 100%
rename from freqtrade/tests/test_hyperopt.py
rename to freqtrade/optimize/hyperopt.py

From b9c4eafd9620fd9d292388186ee1b68719d009e5 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 01:04:11 +0100
Subject: [PATCH 03/12] integrate hyperopt and implement subcommand

---
 freqtrade/misc.py                            |  18 +-
 freqtrade/optimize/__init__.py               |  42 +++-
 freqtrade/optimize/backtesting.py            |  25 +--
 freqtrade/optimize/hyperopt.py               | 199 ++++++++++---------
 freqtrade/tests/__init__.py                  |  21 --
 freqtrade/tests/conftest.py                  |  16 --
 freqtrade/tests/test_misc.py                 |  17 +-
 freqtrade/tests/test_optimize_backtesting.py |  14 +-
 freqtrade/tests/test_optimize_hyperopt.py    |   6 +
 9 files changed, 191 insertions(+), 167 deletions(-)
 create mode 100644 freqtrade/tests/test_optimize_hyperopt.py

diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 784e4903e..5242240e0 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -128,18 +128,20 @@ def parse_args(args: List[str]):
 
 def build_subcommands(parser: argparse.ArgumentParser) -> None:
     """ Builds and attaches all subcommands """
-    from freqtrade.optimize import backtesting
+    from freqtrade.optimize import backtesting, hyperopt
 
     subparsers = parser.add_subparsers(dest='subparser')
-    backtest = subparsers.add_parser('backtesting', help='backtesting module')
-    backtest.set_defaults(func=backtesting.start)
-    backtest.add_argument(
+
+    # Add backtesting subcommand
+    backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
+    backtesting_cmd.set_defaults(func=backtesting.start)
+    backtesting_cmd.add_argument(
         '-l', '--live',
         action='store_true',
         dest='live',
         help='using live data',
     )
-    backtest.add_argument(
+    backtesting_cmd.add_argument(
         '-i', '--ticker-interval',
         help='specify ticker interval in minutes (default: 5)',
         dest='ticker_interval',
@@ -147,13 +149,17 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
         type=int,
         metavar='INT',
     )
-    backtest.add_argument(
+    backtesting_cmd.add_argument(
         '--realistic-simulation',
         help='uses max_open_trades from config to simulate real world limitations',
         action='store_true',
         dest='realistic_simulation',
     )
 
+    # Add hyperopt subcommand
+    hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
+    hyperopt_cmd.set_defaults(func=hyperopt.start)
+
 
 # Required json-schema for user specified config
 CONF_SCHEMA = {
diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py
index 4cffcee10..1bd1a97e8 100644
--- a/freqtrade/optimize/__init__.py
+++ b/freqtrade/optimize/__init__.py
@@ -1 +1,41 @@
-from . import backtesting
+# pragma pylint: disable=missing-docstring
+
+
+import json
+import os
+from typing import Optional, List, Dict
+
+from pandas import DataFrame
+
+from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
+
+
+def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None) -> Dict[str, List]:
+    """
+    Loads ticker history data for the given parameters
+    :param ticker_interval: ticker interval in minutes
+    :param pairs: list of pairs
+    :return: dict
+    """
+    path = os.path.abspath(os.path.dirname(__file__))
+    result = {}
+    _pairs = pairs or [
+        'BTC_BCC', 'BTC_ETH', 'BTC_DASH', 'BTC_POWR', 'BTC_ETC',
+        'BTC_VTC', 'BTC_WAVES', 'BTC_LSK', 'BTC_XLM', 'BTC_OK',
+    ]
+    for pair in _pairs:
+        with open('{abspath}/../tests/testdata/{pair}-{ticker_interval}.json'.format(
+            abspath=path,
+            pair=pair,
+            ticker_interval=ticker_interval,
+        )) as tickerdata:
+            result[pair] = json.load(tickerdata)
+    return result
+
+
+def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
+    """Creates a dataframe and populates indicators for given ticker data"""
+    processed = {}
+    for pair, pair_data in tickerdata.items():
+        processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data))
+    return processed
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index ea861ee84..508d4755d 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -9,34 +9,17 @@ from pandas import DataFrame
 from tabulate import tabulate
 
 from freqtrade import exchange
-from freqtrade.analyze import parse_ticker_dataframe, populate_indicators, \
-    populate_buy_trend, populate_sell_trend
+from freqtrade.analyze import populate_buy_trend, populate_sell_trend
 from freqtrade.exchange import Bittrex
 from freqtrade.main import min_roi_reached
 from freqtrade.misc import load_config
+from freqtrade.optimize import load_data, preprocess
 from freqtrade.persistence import Trade
-from freqtrade.tests import load_backtesting_data
+
 
 logger = logging.getLogger(__name__)
 
 
-def format_results(results: DataFrame):
-    return ('Made {:6d} buys. Average profit {: 5.2f}%. '
-            'Total profit was {: 7.3f}. Average duration {:5.1f} mins.').format(
-                len(results.index),
-                results.profit.mean() * 100.0,
-                results.profit.sum(),
-                results.duration.mean() * 5,
-            )
-
-
-def preprocess(backdata) -> Dict[str, DataFrame]:
-    processed = {}
-    for pair, pair_data in backdata.items():
-        processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data))
-    return processed
-
-
 def get_timeframe(data: Dict[str, Dict]) -> Tuple[arrow.Arrow, arrow.Arrow]:
     """
     Get the maximum timeframe for the given backtest data
@@ -151,7 +134,7 @@ def start(args):
             data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
         print('Using local backtesting data (ignoring whitelist in given config)...')
-        data = load_backtesting_data(args.ticker_interval)
+        data = load_data(args.ticker_interval)
 
     print('Using stake_currency: {} ...\nUsing stake_amount: {} ...'.format(
         config['stake_currency'], config['stake_amount']
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 5face5837..b3c35710b 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -1,29 +1,124 @@
 # pragma pylint: disable=missing-docstring,W0212
-import logging
-import os
+
+
 from functools import reduce
 from math import exp
 from operator import itemgetter
 
-import pytest
 from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
 from pandas import DataFrame
 
-from freqtrade import exchange
+from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
-from freqtrade.optimize.backtesting import backtest, format_results
-from freqtrade.optimize.backtesting import preprocess
-from freqtrade.tests import load_backtesting_data
+from freqtrade.optimize.backtesting import backtest
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
-logging.disable(logging.DEBUG)  # disable debug logs that slow backtesting a lot
-
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
 TOTAL_TRIES = 4
 # pylint: disable=C0103
 current_tries = 0
 
+# Configuration and data used by hyperopt
+PROCESSED = optimize.preprocess(optimize.load_data())
+OPTIMIZE_CONFIG = {
+    'max_open_trades': 3,
+    'stake_currency': 'BTC',
+    'stake_amount': 0.01,
+    'minimal_roi': {
+        '40':  0.0,
+        '30':  0.01,
+        '20':  0.02,
+        '0':  0.04,
+    },
+    'stoploss': -0.10,
+}
+
+SPACE = {
+    'mfi': hp.choice('mfi', [
+        {'enabled': False},
+        {'enabled': True, 'value': hp.quniform('mfi-value', 5, 25, 1)}
+    ]),
+    'fastd': hp.choice('fastd', [
+        {'enabled': False},
+        {'enabled': True, 'value': hp.quniform('fastd-value', 10, 50, 1)}
+    ]),
+    'adx': hp.choice('adx', [
+        {'enabled': False},
+        {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)}
+    ]),
+    'rsi': hp.choice('rsi', [
+        {'enabled': False},
+        {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)}
+    ]),
+    'uptrend_long_ema': hp.choice('uptrend_long_ema', [
+        {'enabled': False},
+        {'enabled': True}
+    ]),
+    'uptrend_short_ema': hp.choice('uptrend_short_ema', [
+        {'enabled': False},
+        {'enabled': True}
+    ]),
+    'over_sar': hp.choice('over_sar', [
+        {'enabled': False},
+        {'enabled': True}
+    ]),
+    'green_candle': hp.choice('green_candle', [
+        {'enabled': False},
+        {'enabled': True}
+    ]),
+    'uptrend_sma': hp.choice('uptrend_sma', [
+        {'enabled': False},
+        {'enabled': True}
+    ]),
+    'trigger': hp.choice('trigger', [
+        {'type': 'lower_bb'},
+        {'type': 'faststoch10'},
+        {'type': 'ao_cross_zero'},
+        {'type': 'ema5_cross_ema10'},
+        {'type': 'macd_cross_signal'},
+        {'type': 'sar_reversal'},
+        {'type': 'stochf_cross'},
+        {'type': 'ht_sine'},
+    ]),
+}
+
+
+def optimizer(params):
+    from freqtrade.optimize import backtesting
+    backtesting.populate_buy_trend = buy_strategy_generator(params)
+
+    results = backtest(OPTIMIZE_CONFIG, PROCESSED)
+
+    result = format_results(results)
+
+    total_profit = results.profit.sum() * 1000
+    trade_count = len(results.index)
+
+    trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
+    profit_loss = max(0, 1 - total_profit / 10000)  # max profit 10000
+
+    # pylint: disable=W0603
+    global current_tries
+    current_tries += 1
+    print('{:5d}/{}: {}'.format(current_tries, TOTAL_TRIES, result))
+
+    return {
+        'loss': trade_loss + profit_loss,
+        'status': STATUS_OK,
+        'result': result
+    }
+
+
+def format_results(results: DataFrame):
+    return ('Made {:6d} buys. Average profit {: 5.2f}%. '
+            'Total profit was {: 7.3f}. Average duration {:5.1f} mins.').format(
+                len(results.index),
+                results.profit.mean() * 100.0,
+                results.profit.sum(),
+                results.duration.mean() * 5,
+            )
+
 
 def buy_strategy_generator(params):
     def populate_buy_trend(dataframe: DataFrame) -> DataFrame:
@@ -70,94 +165,14 @@ def buy_strategy_generator(params):
     return populate_buy_trend
 
 
-@pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set")
-def test_hyperopt(backtest_conf, mocker):
-    mocked_buy_trend = mocker.patch('freqtrade.tests.test_backtesting.populate_buy_trend')
+def start(args):
+    # TODO: parse args
 
-    backdata = load_backtesting_data()
-    processed = preprocess(backdata)
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    def optimizer(params):
-        mocked_buy_trend.side_effect = buy_strategy_generator(params)
-
-        results = backtest(backtest_conf, processed, mocker)
-
-        result = format_results(results)
-
-        total_profit = results.profit.sum() * 1000
-        trade_count = len(results.index)
-
-        trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
-        profit_loss = max(0, 1 - total_profit / 10000)  # max profit 10000
-
-        # pylint: disable=W0603
-        global current_tries
-        current_tries += 1
-        print('{:5d}/{}: {}'.format(current_tries, TOTAL_TRIES, result))
-
-        return {
-            'loss': trade_loss + profit_loss,
-            'status': STATUS_OK,
-            'result': result
-        }
-
-    space = {
-        'mfi': hp.choice('mfi', [
-            {'enabled': False},
-            {'enabled': True, 'value': hp.quniform('mfi-value', 5, 25, 1)}
-        ]),
-        'fastd': hp.choice('fastd', [
-            {'enabled': False},
-            {'enabled': True, 'value': hp.quniform('fastd-value', 10, 50, 1)}
-        ]),
-        'adx': hp.choice('adx', [
-            {'enabled': False},
-            {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)}
-        ]),
-        'rsi': hp.choice('rsi', [
-            {'enabled': False},
-            {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)}
-        ]),
-        'uptrend_long_ema': hp.choice('uptrend_long_ema', [
-            {'enabled': False},
-            {'enabled': True}
-        ]),
-        'uptrend_short_ema': hp.choice('uptrend_short_ema', [
-            {'enabled': False},
-            {'enabled': True}
-        ]),
-        'over_sar': hp.choice('over_sar', [
-            {'enabled': False},
-            {'enabled': True}
-        ]),
-        'green_candle': hp.choice('green_candle', [
-            {'enabled': False},
-            {'enabled': True}
-        ]),
-        'uptrend_sma': hp.choice('uptrend_sma', [
-            {'enabled': False},
-            {'enabled': True}
-        ]),
-        'trigger': hp.choice('trigger', [
-            {'type': 'lower_bb'},
-            {'type': 'faststoch10'},
-            {'type': 'ao_cross_zero'},
-            {'type': 'ema5_cross_ema10'},
-            {'type': 'macd_cross_signal'},
-            {'type': 'sar_reversal'},
-            {'type': 'stochf_cross'},
-            {'type': 'ht_sine'},
-        ]),
-    }
     trials = Trials()
-    best = fmin(fn=optimizer, space=space, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials)
+    best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials)
     print('\n\n\n\n==================== HYPEROPT BACKTESTING REPORT ==============================')
     print('Best parameters {}'.format(best))
     newlist = sorted(trials.results, key=itemgetter('loss'))
     print('Result: {}'.format(newlist[0]['result']))
-
-
-if __name__ == '__main__':
-    # for profiling with cProfile and line_profiler
-    pytest.main([__file__, '-s'])
diff --git a/freqtrade/tests/__init__.py b/freqtrade/tests/__init__.py
index ebebe7c98..e69de29bb 100644
--- a/freqtrade/tests/__init__.py
+++ b/freqtrade/tests/__init__.py
@@ -1,21 +0,0 @@
-# pragma pylint: disable=missing-docstring
-import json
-import os
-from typing import Optional, List
-
-
-def load_backtesting_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None):
-    path = os.path.abspath(os.path.dirname(__file__))
-    result = {}
-    _pairs = pairs or [
-        'BTC_BCC', 'BTC_ETH', 'BTC_DASH', 'BTC_POWR', 'BTC_ETC',
-        'BTC_VTC', 'BTC_WAVES', 'BTC_LSK', 'BTC_XLM', 'BTC_OK',
-    ]
-    for pair in _pairs:
-        with open('{abspath}/testdata/{pair}-{ticker_interval}.json'.format(
-            abspath=path,
-            pair=pair,
-            ticker_interval=ticker_interval,
-        )) as tickerdata:
-            result[pair] = json.load(tickerdata)
-    return result
diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py
index e624e96c7..f2a9362ec 100644
--- a/freqtrade/tests/conftest.py
+++ b/freqtrade/tests/conftest.py
@@ -51,22 +51,6 @@ def default_conf():
     return configuration
 
 
-@pytest.fixture(scope="module")
-def backtest_conf():
-    return {
-        "max_open_trades": 3,
-        "stake_currency": "BTC",
-        "stake_amount": 0.01,
-        "minimal_roi": {
-            "40":  0.0,
-            "30":  0.01,
-            "20":  0.02,
-            "0":  0.04
-        },
-        "stoploss": -0.10
-    }
-
-
 @pytest.fixture
 def update():
     _update = Update(0)
diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py
index 9748deeea..1c72a70ba 100644
--- a/freqtrade/tests/test_misc.py
+++ b/freqtrade/tests/test_misc.py
@@ -78,10 +78,10 @@ def test_parse_args_backtesting(mocker):
 
 def test_parse_args_backtesting_invalid():
     with pytest.raises(SystemExit, match=r'2'):
-        parse_args(['--ticker-interval'])
+        parse_args(['backtesting --ticker-interval'])
 
     with pytest.raises(SystemExit, match=r'2'):
-        parse_args(['--ticker-interval', 'abc'])
+        parse_args(['backtesting --ticker-interval', 'abc'])
 
 
 def test_parse_args_backtesting_custom(mocker):
@@ -99,6 +99,19 @@ def test_parse_args_backtesting_custom(mocker):
     assert call_args.ticker_interval == 1
 
 
+def test_parse_args_hyperopt(mocker):
+    hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock())
+    args = parse_args(['hyperopt'])
+    assert args is None
+    assert hyperopt_mock.call_count == 1
+
+    call_args = hyperopt_mock.call_args[0][0]
+    assert call_args.config == 'config.json'
+    assert call_args.loglevel == 20
+    assert call_args.subparser == 'hyperopt'
+    assert call_args.func is not None
+
+
 def test_load_config(default_conf, mocker):
     file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open(
         read_data=json.dumps(default_conf)
diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py
index 36f4cd144..7986f6d35 100644
--- a/freqtrade/tests/test_optimize_backtesting.py
+++ b/freqtrade/tests/test_optimize_backtesting.py
@@ -1,18 +1,16 @@
 # pragma pylint: disable=missing-docstring,W0212
 
 
-from freqtrade import exchange
+from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
-from freqtrade.optimize.backtesting import backtest, preprocess
-from freqtrade.tests import load_backtesting_data
+from freqtrade.optimize.backtesting import backtest
 
 
-def test_backtest(backtest_conf, mocker):
-    mocker.patch.dict('freqtrade.main._CONF', backtest_conf)
+def test_backtest(default_conf, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    data = load_backtesting_data(ticker_interval=5, pairs=['BTC_ETH'])
-    results = backtest(backtest_conf, preprocess(data), 10, True)
+    data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH'])
+    results = backtest(default_conf, optimize.preprocess(data), 10, True)
     num_resutls = len(results)
     assert num_resutls > 0
-
diff --git a/freqtrade/tests/test_optimize_hyperopt.py b/freqtrade/tests/test_optimize_hyperopt.py
new file mode 100644
index 000000000..a8bfe7dd4
--- /dev/null
+++ b/freqtrade/tests/test_optimize_hyperopt.py
@@ -0,0 +1,6 @@
+# pragma pylint: disable=missing-docstring,W0212
+
+
+def test_optimizer(default_conf, mocker):
+    # TODO: implement test
+    pass

From 9ff1f05e66c6ae52c5aaf59924121dde153f9c92 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 01:12:44 +0100
Subject: [PATCH 04/12] add --epochs to hyperopt subcommand

---
 freqtrade/misc.py                 |  9 ++++++++-
 freqtrade/optimize/backtesting.py |  1 +
 freqtrade/optimize/hyperopt.py    | 18 +++++++++++-------
 freqtrade/tests/test_misc.py      | 14 ++++++++++++++
 4 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 5242240e0..4f09db4c4 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -159,7 +159,14 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
     # Add hyperopt subcommand
     hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
     hyperopt_cmd.set_defaults(func=hyperopt.start)
-
+    hyperopt_cmd.add_argument(
+        '-e', '--epochs',
+        help='specify number of epochs (default: 100)',
+        dest='epochs',
+        default=100,
+        type=int,
+        metavar='INT',
+    )
 
 # Required json-schema for user specified config
 CONF_SCHEMA = {
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 508d4755d..436be4ecc 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -151,6 +151,7 @@ def start(args):
         print('Using max_open_trades: {} ...'.format(config['max_open_trades']))
         max_open_trades = config['max_open_trades']
 
+    # Monkey patch config
     from freqtrade import main
     main._CONF = config
 
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index b3c35710b..66e36695e 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -16,8 +16,7 @@ from freqtrade.vendor.qtpylib.indicators import crossed_above
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
 TOTAL_TRIES = 4
-# pylint: disable=C0103
-current_tries = 0
+_CURRENT_TRIES = 0
 
 # Configuration and data used by hyperopt
 PROCESSED = optimize.preprocess(optimize.load_data())
@@ -85,6 +84,8 @@ SPACE = {
 
 
 def optimizer(params):
+    global _CURRENT_TRIES
+
     from freqtrade.optimize import backtesting
     backtesting.populate_buy_trend = buy_strategy_generator(params)
 
@@ -98,10 +99,8 @@ def optimizer(params):
     trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
     profit_loss = max(0, 1 - total_profit / 10000)  # max profit 10000
 
-    # pylint: disable=W0603
-    global current_tries
-    current_tries += 1
-    print('{:5d}/{}: {}'.format(current_tries, TOTAL_TRIES, result))
+    _CURRENT_TRIES += 1
+    print('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result))
 
     return {
         'loss': trade_loss + profit_loss,
@@ -166,7 +165,12 @@ def buy_strategy_generator(params):
 
 
 def start(args):
-    # TODO: parse args
+    global TOTAL_TRIES
+    TOTAL_TRIES = args.epochs
+
+    # Monkey patch config
+    from freqtrade import main
+    main._CONF = OPTIMIZE_CONFIG
 
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py
index 1c72a70ba..6f52b44b8 100644
--- a/freqtrade/tests/test_misc.py
+++ b/freqtrade/tests/test_misc.py
@@ -112,6 +112,20 @@ def test_parse_args_hyperopt(mocker):
     assert call_args.func is not None
 
 
+def test_parse_args_hyperopt_custom(mocker):
+    hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock())
+    args = parse_args(['-c', 'test_conf.json', 'hyperopt', '--epochs', '20'])
+    assert args is None
+    assert hyperopt_mock.call_count == 1
+
+    call_args = hyperopt_mock.call_args[0][0]
+    assert call_args.config == 'test_conf.json'
+    assert call_args.epochs == 20
+    assert call_args.loglevel == 20
+    assert call_args.subparser == 'hyperopt'
+    assert call_args.func is not None
+
+
 def test_load_config(default_conf, mocker):
     file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open(
         read_data=json.dumps(default_conf)

From 7f3f127165f48487e0b920d48022d726fa5d6660 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 01:13:28 +0100
Subject: [PATCH 05/12] remove custom env from .travis.yml

---
 .travis.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1d6a6a9b6..461324f45 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,9 +4,6 @@ os:
 language: python
 python:
 - 3.6
-env:
-  - BACKTEST=
-  - BACKTEST=true
 addons:
   apt:
     packages:

From a23fce519d29a7eabf0acaf38b150641c3470cd1 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 01:22:36 +0100
Subject: [PATCH 06/12] pretty print hyperopt results

---
 freqtrade/optimize/hyperopt.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 66e36695e..e48dc6ec3 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -1,9 +1,12 @@
 # pragma pylint: disable=missing-docstring,W0212
 
 
+import json
+
 from functools import reduce
 from math import exp
 from operator import itemgetter
+from pprint import pprint
 
 from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
 from pandas import DataFrame
@@ -176,7 +179,7 @@ def start(args):
 
     trials = Trials()
     best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials)
-    print('\n\n\n\n==================== HYPEROPT BACKTESTING REPORT ==============================')
-    print('Best parameters {}'.format(best))
-    newlist = sorted(trials.results, key=itemgetter('loss'))
-    print('Result: {}'.format(newlist[0]['result']))
+    print('\n==================== HYPEROPT BACKTESTING REPORT ==============================\n')
+    print('Best parameters: {}'.format(json.dumps(best, indent=4)))
+    results = sorted(trials.results, key=itemgetter('loss'))
+    print('Best Result: {}\n'.format(results[0]['result']))

From 5bf583cba402b27874653b46379c579846fe2b11 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 01:23:18 +0100
Subject: [PATCH 07/12] remove unused imports

---
 freqtrade/optimize/backtesting.py | 1 -
 freqtrade/optimize/hyperopt.py    | 2 --
 2 files changed, 3 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 436be4ecc..6c81fc75e 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -16,7 +16,6 @@ from freqtrade.misc import load_config
 from freqtrade.optimize import load_data, preprocess
 from freqtrade.persistence import Trade
 
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index e48dc6ec3..b243c0981 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -2,11 +2,9 @@
 
 
 import json
-
 from functools import reduce
 from math import exp
 from operator import itemgetter
-from pprint import pprint
 
 from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
 from pandas import DataFrame

From e27a6a7a91772f73fc935c05ad77c477aa1ef1c6 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 02:04:37 +0100
Subject: [PATCH 08/12] add mongodb support for hyperopt parallelization

---
 .gitignore                        |  1 +
 freqtrade/misc.py                 |  7 ++++++
 freqtrade/optimize/backtesting.py | 32 ++++++++++++++-----------
 freqtrade/optimize/hyperopt.py    | 40 ++++++++++++++++++++++++-------
 scripts/start-hyperopt-worker.sh  |  5 ++++
 scripts/start-mongodb.sh          | 12 ++++++++++
 6 files changed, 74 insertions(+), 23 deletions(-)
 create mode 100755 scripts/start-hyperopt-worker.sh
 create mode 100755 scripts/start-mongodb.sh

diff --git a/.gitignore b/.gitignore
index 2d4af4dac..00f0e4651 100644
--- a/.gitignore
+++ b/.gitignore
@@ -76,6 +76,7 @@ target/
 config.json
 preprocessor.py
 *.sqlite
+.mongodb
 
 .env
 .venv
diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 4f09db4c4..dcae19ac4 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -167,6 +167,13 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
         type=int,
         metavar='INT',
     )
+    hyperopt_cmd.add_argument(
+        '--use-mongodb',
+        help='parallelize evaluations with mongodb (requires mongod in PATH)',
+        dest='mongodb',
+        action='store_true',
+    )
+
 
 # Required json-schema for user specified config
 CONF_SCHEMA = {
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 6c81fc75e..49f7e79a8 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -118,36 +118,38 @@ def backtest(config: Dict, processed: Dict[str, DataFrame],
 
 
 def start(args):
-    print('')
+    # Initialize logger
+    logging.basicConfig(
+        level=args.loglevel,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    )
+
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    print('Using config: {} ...'.format(args.config))
+    logger.info('Using config: %s ...', args.config)
     config = load_config(args.config)
 
-    print('Using ticker_interval: {} ...'.format(args.ticker_interval))
+    logger.info('Using ticker_interval: %s ...', args.ticker_interval)
 
     data = {}
     if args.live:
-        print('Downloading data for all pairs in whitelist ...')
+        logger.info('Downloading data for all pairs in whitelist ...')
         for pair in config['exchange']['pair_whitelist']:
             data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
-        print('Using local backtesting data (ignoring whitelist in given config)...')
+        logger.info('Using local backtesting data (ignoring whitelist in given config) ...')
         data = load_data(args.ticker_interval)
 
-    print('Using stake_currency: {} ...\nUsing stake_amount: {} ...'.format(
-        config['stake_currency'], config['stake_amount']
-    ))
+        logger.info('Using stake_currency: %s ...', config['stake_currency'])
+        logger.info('Using stake_amount: %s ...', config['stake_amount'])
 
     # Print timeframe
     min_date, max_date = get_timeframe(data)
-    print('Measuring data from {} up to {} ...'.format(
-        min_date.isoformat(), max_date.isoformat()
-    ))
+    logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat())
 
     max_open_trades = 0
     if args.realistic_simulation:
-        print('Using max_open_trades: {} ...'.format(config['max_open_trades']))
+        logger.info('Using max_open_trades: %s ...', config['max_open_trades'])
         max_open_trades = config['max_open_trades']
 
     # Monkey patch config
@@ -158,5 +160,7 @@ def start(args):
     results = backtest(
         config, preprocess(data), max_open_trades, args.realistic_simulation
     )
-    print('====================== BACKTESTING REPORT ======================================\n\n')
-    print(generate_text_table(data, results, config['stake_currency']))
+    logger.info(
+        '\n====================== BACKTESTING REPORT ======================================\n%s',
+        generate_text_table(data, results, config['stake_currency'])
+    )
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index b243c0981..1b7e9a225 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -2,11 +2,13 @@
 
 
 import json
+import logging
 from functools import reduce
 from math import exp
 from operator import itemgetter
 
 from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
+from hyperopt.mongoexp import MongoTrials
 from pandas import DataFrame
 
 from freqtrade import exchange, optimize
@@ -14,6 +16,9 @@ from freqtrade.exchange import Bittrex
 from freqtrade.optimize.backtesting import backtest
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
+logger = logging.getLogger(__name__)
+
+
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
 TOTAL_TRIES = 4
@@ -34,6 +39,11 @@ OPTIMIZE_CONFIG = {
     'stoploss': -0.10,
 }
 
+# Monkey patch config
+from freqtrade import main
+main._CONF = OPTIMIZE_CONFIG
+
+
 SPACE = {
     'mfi': hp.choice('mfi', [
         {'enabled': False},
@@ -101,7 +111,7 @@ def optimizer(params):
     profit_loss = max(0, 1 - total_profit / 10000)  # max profit 10000
 
     _CURRENT_TRIES += 1
-    print('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result))
+    logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result))
 
     return {
         'loss': trade_loss + profit_loss,
@@ -169,15 +179,27 @@ def start(args):
     global TOTAL_TRIES
     TOTAL_TRIES = args.epochs
 
-    # Monkey patch config
-    from freqtrade import main
-    main._CONF = OPTIMIZE_CONFIG
-
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    trials = Trials()
+    # Initialize logger
+    logging.basicConfig(
+        level=args.loglevel,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    )
+
+    if args.mongodb:
+        logger.info('Using mongodb.')
+        logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually')
+
+        db_name = 'freqtrade_hyperopt'
+        trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
+    else:
+        trials = Trials()
+
     best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials)
-    print('\n==================== HYPEROPT BACKTESTING REPORT ==============================\n')
-    print('Best parameters: {}'.format(json.dumps(best, indent=4)))
+    logger.info(
+        '\n==================== HYPEROPT BACKTESTING REPORT ==============================\n'
+    )
+    logger.info('Best parameters:\n%s', json.dumps(best, indent=4))
     results = sorted(trials.results, key=itemgetter('loss'))
-    print('Best Result: {}\n'.format(results[0]['result']))
+    logger.info('Best Result:\n%s', results[0]['result'])
diff --git a/scripts/start-hyperopt-worker.sh b/scripts/start-hyperopt-worker.sh
new file mode 100755
index 000000000..ffe4753cb
--- /dev/null
+++ b/scripts/start-hyperopt-worker.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+DB_NAME=freqtrade_hyperopt
+
+hyperopt-mongo-worker --mongo=127.0.0.1:1234/${DB_NAME} --poll-interval=0.1
diff --git a/scripts/start-mongodb.sh b/scripts/start-mongodb.sh
new file mode 100755
index 000000000..5a1758d04
--- /dev/null
+++ b/scripts/start-mongodb.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DB_PATH="${DIR}/../.mongodb"
+
+mkdir -p ${DB_PATH}
+mongod --dbpath ${DB_PATH} \
+    --bind_ip 127.0.0.1 \
+    --port 1234 \
+    --directoryperdb \
+    --journal \
+    --nohttpinterface

From 2fe11cd77ac45e3291405ab09dc6e289d2a493b5 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 03:28:18 +0100
Subject: [PATCH 09/12] add helper scripts for mongodb

---
 scripts/start-hyperopt-worker.sh | 11 +++++++++--
 scripts/start-mongodb.sh         |  6 +++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/scripts/start-hyperopt-worker.sh b/scripts/start-hyperopt-worker.sh
index ffe4753cb..1b0d240ca 100755
--- a/scripts/start-hyperopt-worker.sh
+++ b/scripts/start-hyperopt-worker.sh
@@ -1,5 +1,12 @@
-#!/bin/bash
+#!/bin/bash -e
 
 DB_NAME=freqtrade_hyperopt
 
-hyperopt-mongo-worker --mongo=127.0.0.1:1234/${DB_NAME} --poll-interval=0.1
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+WORK_DIR="${DIR}/../.hyperopt/worker"
+
+mkdir -p "${WORK_DIR}"
+hyperopt-mongo-worker \
+    --mongo="127.0.0.1:1234/${DB_NAME}" \
+    --poll-interval=0.1 \
+    --workdir="${WORK_DIR}"
\ No newline at end of file
diff --git a/scripts/start-mongodb.sh b/scripts/start-mongodb.sh
index 5a1758d04..02b6624f4 100755
--- a/scripts/start-mongodb.sh
+++ b/scripts/start-mongodb.sh
@@ -1,10 +1,10 @@
 #!/bin/bash -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-DB_PATH="${DIR}/../.mongodb"
+DB_PATH="${DIR}/../.hyperopt/mongodb"
 
-mkdir -p ${DB_PATH}
-mongod --dbpath ${DB_PATH} \
+mkdir -p "${DB_PATH}"
+mongod --dbpath "${DB_PATH}" \
     --bind_ip 127.0.0.1 \
     --port 1234 \
     --directoryperdb \

From 68521ea46cefdec0573ec20f9712e7372ed7a252 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 03:28:39 +0100
Subject: [PATCH 10/12] adapt README

---
 .gitignore |  3 ++-
 README.md  | 30 ++++++++++++++++++++++--------
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index 00f0e4651..9b0606756 100644
--- a/.gitignore
+++ b/.gitignore
@@ -76,7 +76,8 @@ target/
 config.json
 preprocessor.py
 *.sqlite
-.mongodb
+.hyperopt
+logfile.txt
 
 .env
 .venv
diff --git a/README.md b/README.md
index 398056079..28cba0855 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ See the example below:
     "40": 0.0,    # Sell after 40 minutes if the profit is not negative
     "30": 0.01,   # Sell after 30 minutes if there is at least 1% profit
     "20": 0.02,   # Sell after 20 minutes if there is at least 2% profit
-    "0":  0.04  # Sell immediately if there is at least 4% profit
+    "0":  0.04    # Sell immediately if there is at least 4% profit
 },
 ```
 
@@ -164,25 +164,39 @@ optional arguments:
 Backtesting also uses the config specified via `-c/--config`.
 
 ```
-usage: freqtrade backtesting [-h] [-l] [-i INT]
+usage: freqtrade backtesting [-h] [-l] [-i INT] [--realistic-simulation]
 
 optional arguments:
   -h, --help            show this help message and exit
   -l, --live            using live data
   -i INT, --ticker-interval INT
                         specify ticker interval in minutes (default: 5)
+  --realistic-simulation
+                        uses max_open_trades from config to simulate real
+                        world limitations
+
 ```
 
+### Hyperopt
+
+It is possible to use hyperopt for trading strategy optimization.
+Hyperopt uses an internal config named `OPTIMIZE_CONFIG` located in `freqtrade/optimize/hyperopt.py`.
+
+```
+usage: freqtrade hyperopt [-h] [-e INT] [--use-mongodb]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -e INT, --epochs INT  specify number of epochs (default: 100)
+  --use-mongodb         parallelize evaluations with mongodb (requires mongod
+                        in PATH)
+
+```
 
 ### Execute tests
 
 ```
-$ pytest
-```
-This will by default skip the slow running backtest set. To run backtest set:
-
-```
-$ BACKTEST=true pytest -s freqtrade/tests/test_backtesting.py
+$ pytest freqtrade
 ```
 
 ### Contributing

From 0c35e6ad1920677a31d29b2ddc0f78766ffb873d Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 03:28:52 +0100
Subject: [PATCH 11/12] minor changes

---
 freqtrade/analyze.py           |  1 +
 freqtrade/optimize/hyperopt.py | 12 ++++++------
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py
index 8edf86340..d586077db 100644
--- a/freqtrade/analyze.py
+++ b/freqtrade/analyze.py
@@ -15,6 +15,7 @@ from freqtrade.vendor.qtpylib.indicators import awesome_oscillator, crossed_abov
 
 logger = logging.getLogger(__name__)
 
+
 class SignalType(Enum):
     """ Enum to distinguish between buy and sell signals """
     BUY = "buy"
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 1b7e9a225..36eb0d275 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -16,12 +16,15 @@ from freqtrade.exchange import Bittrex
 from freqtrade.optimize.backtesting import backtest
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
+# Remove noisy log messages
+logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING)
+
 logger = logging.getLogger(__name__)
 
 
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
-TOTAL_TRIES = 4
+TOTAL_TRIES = None
 _CURRENT_TRIES = 0
 
 # Configuration and data used by hyperopt
@@ -188,8 +191,8 @@ def start(args):
     )
 
     if args.mongodb:
-        logger.info('Using mongodb.')
-        logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually')
+        logger.info('Using mongodb ...')
+        logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')
 
         db_name = 'freqtrade_hyperopt'
         trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
@@ -197,9 +200,6 @@ def start(args):
         trials = Trials()
 
     best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials)
-    logger.info(
-        '\n==================== HYPEROPT BACKTESTING REPORT ==============================\n'
-    )
     logger.info('Best parameters:\n%s', json.dumps(best, indent=4))
     results = sorted(trials.results, key=itemgetter('loss'))
     logger.info('Best Result:\n%s', results[0]['result'])

From 0c9993cc8948c4732696a21bfdb25b5d8a2e6523 Mon Sep 17 00:00:00 2001
From: gcarq <michael.egger@tsn.at>
Date: Sat, 25 Nov 2017 15:40:19 +0100
Subject: [PATCH 12/12] convert bash scripts to python scripts

---
 scripts/start-hyperopt-worker.py | 27 +++++++++++++++++++++++++++
 scripts/start-hyperopt-worker.sh | 12 ------------
 scripts/start-mongodb.py         | 21 +++++++++++++++++++++
 scripts/start-mongodb.sh         | 12 ------------
 4 files changed, 48 insertions(+), 24 deletions(-)
 create mode 100755 scripts/start-hyperopt-worker.py
 delete mode 100755 scripts/start-hyperopt-worker.sh
 create mode 100755 scripts/start-mongodb.py
 delete mode 100755 scripts/start-mongodb.sh

diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py
new file mode 100755
index 000000000..8b0ae6326
--- /dev/null
+++ b/scripts/start-hyperopt-worker.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+import multiprocessing
+import os
+import subprocess
+
+PROC_COUNT = multiprocessing.cpu_count() - 1
+DB_NAME = 'freqtrade_hyperopt'
+WORK_DIR = os.path.join(
+    os.path.sep,
+    os.path.abspath(os.path.dirname(__file__)),
+    '..', '.hyperopt', 'worker'
+)
+if not os.path.exists(WORK_DIR):
+    os.makedirs(WORK_DIR)
+
+# Spawn workers
+command = [
+    'hyperopt-mongo-worker',
+    '--mongo=127.0.0.1:1234/{}'.format(DB_NAME),
+    '--poll-interval=0.1',
+    '--workdir={}'.format(WORK_DIR),
+]
+processes = [subprocess.Popen(command) for i in range(PROC_COUNT)]
+
+# Join all workers
+for proc in processes:
+    proc.wait()
diff --git a/scripts/start-hyperopt-worker.sh b/scripts/start-hyperopt-worker.sh
deleted file mode 100755
index 1b0d240ca..000000000
--- a/scripts/start-hyperopt-worker.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash -e
-
-DB_NAME=freqtrade_hyperopt
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-WORK_DIR="${DIR}/../.hyperopt/worker"
-
-mkdir -p "${WORK_DIR}"
-hyperopt-mongo-worker \
-    --mongo="127.0.0.1:1234/${DB_NAME}" \
-    --poll-interval=0.1 \
-    --workdir="${WORK_DIR}"
\ No newline at end of file
diff --git a/scripts/start-mongodb.py b/scripts/start-mongodb.py
new file mode 100755
index 000000000..910ee9233
--- /dev/null
+++ b/scripts/start-mongodb.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+
+
+DB_PATH = os.path.join(
+    os.path.sep,
+    os.path.abspath(os.path.dirname(__file__)),
+    '..', '.hyperopt', 'mongodb'
+)
+if not os.path.exists(DB_PATH):
+    os.makedirs(DB_PATH)
+
+subprocess.Popen([
+    'mongod',
+    '--bind_ip=127.0.0.1',
+    '--port=1234',
+    '--nohttpinterface',
+    '--dbpath={}'.format(DB_PATH),
+]).wait()
diff --git a/scripts/start-mongodb.sh b/scripts/start-mongodb.sh
deleted file mode 100755
index 02b6624f4..000000000
--- a/scripts/start-mongodb.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-DB_PATH="${DIR}/../.hyperopt/mongodb"
-
-mkdir -p "${DB_PATH}"
-mongod --dbpath "${DB_PATH}" \
-    --bind_ip 127.0.0.1 \
-    --port 1234 \
-    --directoryperdb \
-    --journal \
-    --nohttpinterface