Merge pull request #242 from gcarq/backtesting-unittests

Backtesting and hyperopt unit tests

freqtrade/optimize/__init__.py

@@ -4,11 +4,9 @@ import logging
 import json
 import os
 from typing import Optional, List, Dict
+from pandas import DataFrame
 from freqtrade.exchange import get_ticker_history
 from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf
-
-from pandas import DataFrame
-
 from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
 
 logger = logging.getLogger(__name__)
@@ -50,10 +48,8 @@ def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None,
 
 def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
     """Creates a dataframe and populates indicators for given ticker data"""
-    processed = {}
+    return {pair: populate_indicators(parse_ticker_dataframe(pair_data))
-    for pair, pair_data in tickerdata.items():
+            for pair, pair_data in tickerdata.items()}
-        processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data))
-    return processed
 
 
 def testdata_path() -> str:

freqtrade/optimize/backtesting.py

@@ -111,14 +111,14 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame],
 
                 if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1:
                     current_profit_percent = trade.calc_profit_percent(rate=row2.close)
-                    current_profit_BTC = trade.calc_profit(rate=row2.close)
+                    current_profit_btc = trade.calc_profit(rate=row2.close)
                     lock_pair_until = row2.Index
 
                     trades.append(
                         (
                             pair,
                             current_profit_percent,
-                            current_profit_BTC,
+                            current_profit_btc,
                             row2.Index - row.Index
                         )
                     )

freqtrade/optimize/hyperopt.py

@@ -25,12 +25,10 @@ logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
 
 logger = logging.getLogger(__name__)
 
-
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
 TOTAL_TRIES = None
 _CURRENT_TRIES = 0
-
 CURRENT_BEST_LOSS = 100
 
 # this is expexted avg profit * expected trade count
@@ -111,6 +109,13 @@ def log_results(results):
         sys.stdout.flush()
 
 
+def calculate_loss(total_profit: float, trade_count: int):
+    """ objective function, returns smaller number for more optimal results """
+    trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
+    profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT)
+    return trade_loss + profit_loss
+
+
 def optimizer(params):
     global _CURRENT_TRIES
 
@@ -118,37 +123,33 @@ def optimizer(params):
     backtesting.populate_buy_trend = buy_strategy_generator(params)
 
     results = backtest(OPTIMIZE_CONFIG['stake_amount'], PROCESSED)
-
+    result_explanation = format_results(results)
-    result = format_results(results)
 
     total_profit = results.profit_percent.sum()
     trade_count = len(results.index)
 
     if trade_count == 0:
+        print('.', end='')
         return {
             'status': STATUS_FAIL,
             'loss': float('inf')
         }
 
-    trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
+    loss = calculate_loss(total_profit, trade_count)
-    profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT)
+
-    loss = trade_loss + profit_loss
     _CURRENT_TRIES += 1
 
-    result_data = {
+    log_results({
         'loss': loss,
         'current_tries': _CURRENT_TRIES,
         'total_tries': TOTAL_TRIES,
-        'result': result,
+        'result': result_explanation,
-    }
+    })
-    log_results(result_data)
 
     return {
         'loss': loss,
         'status': STATUS_OK,
-        'result': result,
+        'result': result_explanation,
-        'total_profit': total_profit,
-        'avg_profit': results.profit_percent.mean() * 100.0,
     }
 
 

freqtrade/tests/optimize/test_backtesting.py

@@ -1,12 +1,36 @@
 # pragma pylint: disable=missing-docstring,W0212
 
 import os
+import pandas as pd
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
-from freqtrade.optimize.backtesting import backtest
+from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe
 from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata
 
 
+def test_generate_text_table():
+    results = pd.DataFrame(
+        {
+            'currency': ['BTC_ETH', 'BTC_ETH'],
+            'profit_percent': [0.1, 0.2],
+            'profit_BTC': [0.2, 0.4],
+            'duration': [10, 30]
+        }
+    )
+    assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == (
+        'pair       buy count  avg profit    total profit      avg duration\n'
+        '-------  -----------  ------------  --------------  --------------\n'
+        'BTC_ETH            2  15.00%        0.60000000 BTC             100\n'
+        'TOTAL              2  15.00%        0.60000000 BTC             100')
+
+
+def test_get_timeframe():
+    data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST'])
+    min_date, max_date = get_timeframe(data)
+    assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
+    assert max_date.isoformat() == '2017-11-14T22:59:00+00:00'
+
+
 def test_backtest(default_conf, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     exchange._API = Bittrex({'key': '', 'secret': ''})

freqtrade/tests/optimize/test_hyperopt.py

@@ -0,0 +1,79 @@
+# pragma pylint: disable=missing-docstring,W0212,C0103
+
+from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \
+    log_results
+
+
+def test_loss_calculation_prefer_correct_trade_count():
+    correct = calculate_loss(1, TARGET_TRADES)
+    over = calculate_loss(1, TARGET_TRADES + 100)
+    under = calculate_loss(1, TARGET_TRADES - 100)
+    assert over > correct
+    assert under > correct
+
+
+def test_loss_calculation_has_limited_profit():
+    correct = calculate_loss(EXPECTED_MAX_PROFIT, TARGET_TRADES)
+    over = calculate_loss(EXPECTED_MAX_PROFIT * 2, TARGET_TRADES)
+    under = calculate_loss(EXPECTED_MAX_PROFIT / 2, TARGET_TRADES)
+    assert over == correct
+    assert under > correct
+
+
+def create_trials(mocker):
+    return mocker.Mock(
+        results=[{
+            'loss': 1,
+            'result': 'foo'
+        }]
+    )
+
+
+def test_start_calls_fmin(mocker):
+    mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker))
+    mocker.patch('freqtrade.optimize.preprocess')
+    mocker.patch('freqtrade.optimize.load_data')
+    mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
+
+    args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False)
+    start(args)
+
+    mock_fmin.assert_called_once()
+
+
+def test_start_uses_mongotrials(mocker):
+    mock_mongotrials = mocker.patch('freqtrade.optimize.hyperopt.MongoTrials',
+                                    return_value=create_trials(mocker))
+    mocker.patch('freqtrade.optimize.preprocess')
+    mocker.patch('freqtrade.optimize.load_data')
+    mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
+
+    args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True)
+    start(args)
+
+    mock_mongotrials.assert_called_once()
+
+
+def test_log_results_if_loss_improves(mocker):
+    logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info')
+    global CURRENT_BEST_LOSS
+    CURRENT_BEST_LOSS = 2
+    log_results({
+        'loss': 1,
+        'current_tries': 1,
+        'total_tries': 2,
+        'result': 'foo'
+    })
+
+    logger.assert_called_once()
+
+
+def test_no_log_if_loss_does_not_improve(mocker):
+    logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info')
+    global CURRENT_BEST_LOSS
+    CURRENT_BEST_LOSS = 2
+    log_results({
+        'loss': 3,
+    })
+
+    assert not logger.called

freqtrade/tests/test_optimize_hyperopt.py

@@ -1,6 +0,0 @@
-# pragma pylint: disable=missing-docstring,W0212
-
-
-def test_optimizer(default_conf, mocker):
-    # TODO: implement test
-    pass