From 9b4c0f01f2b19ad556f337d290e1823e8f0014f7 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 08:05:49 +0200 Subject: [PATCH 1/5] more unit tests for backtesting --- freqtrade/tests/test_optimize_backtesting.py | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py index 558cfd8d1..99f95d9bf 100644 --- a/freqtrade/tests/test_optimize_backtesting.py +++ b/freqtrade/tests/test_optimize_backtesting.py @@ -1,12 +1,36 @@ # pragma pylint: disable=missing-docstring,W0212 import os +import pandas as pd from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex -from freqtrade.optimize.backtesting import backtest +from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata +def test_generate_text_table(): + results = pd.DataFrame( + { + 'currency': ['BTC_ETH', 'BTC_ETH'], + 'profit_percent': [0.1, 0.2], + 'profit_BTC': [0.2, 0.4], + 'duration': [10, 30] + } + ) + assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == ( + 'pair buy count avg profit total profit avg duration\n' + '------- ----------- ------------ -------------- --------------\n' + 'BTC_ETH 2 15.00% 0.60000000 BTC 100\n' + 'TOTAL 2 15.00% 0.60000000 BTC 100') + + +def test_get_timeframe(): + data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) + min_date, max_date = get_timeframe(data) + assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' + assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' + + def test_backtest(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) exchange._API = Bittrex({'key': '', 'secret': ''}) From ae0a1436e2a7bbcd5903e558f0dec87ba398b161 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 18:54:14 +0200 Subject: [PATCH 2/5] match test files to prod files for backtesting/hyperopt --- .../test_backtesting.py} | 0 .../{test_optimize_hyperopt.py => optimize/test_hyperopt.py} | 0 .../test_hyperopt_config.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename freqtrade/tests/{test_optimize_backtesting.py => optimize/test_backtesting.py} (100%) rename freqtrade/tests/{test_optimize_hyperopt.py => optimize/test_hyperopt.py} (100%) rename freqtrade/tests/{test_optimize_hyperopt_config.py => optimize/test_hyperopt_config.py} (100%) diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py similarity index 100% rename from freqtrade/tests/test_optimize_backtesting.py rename to freqtrade/tests/optimize/test_backtesting.py diff --git a/freqtrade/tests/test_optimize_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py similarity index 100% rename from freqtrade/tests/test_optimize_hyperopt.py rename to freqtrade/tests/optimize/test_hyperopt.py diff --git a/freqtrade/tests/test_optimize_hyperopt_config.py b/freqtrade/tests/optimize/test_hyperopt_config.py similarity index 100% rename from freqtrade/tests/test_optimize_hyperopt_config.py rename to freqtrade/tests/optimize/test_hyperopt_config.py From 7b0beb0afa0de4beb545452431499c77162db5f9 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 09:59:38 +0200 Subject: [PATCH 3/5] cleanups --- freqtrade/optimize/__init__.py | 10 +++------- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/optimize/hyperopt.py | 14 +++++--------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index a77576a27..ac077506a 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -4,11 +4,9 @@ import logging import json import os from typing import Optional, List, Dict +from pandas import DataFrame from freqtrade.exchange import get_ticker_history from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf - -from pandas import DataFrame - from freqtrade.analyze import populate_indicators, parse_ticker_dataframe logger = logging.getLogger(__name__) @@ -50,10 +48,8 @@ def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None, def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: """Creates a dataframe and populates indicators for given ticker data""" - processed = {} - for pair, pair_data in tickerdata.items(): - processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data)) - return processed + return {pair: populate_indicators(parse_ticker_dataframe(pair_data)) + for pair, pair_data in tickerdata.items()} def testdata_path() -> str: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2684d2576..984ca3e72 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -111,14 +111,14 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1: current_profit_percent = trade.calc_profit_percent(rate=row2.close) - current_profit_BTC = trade.calc_profit(rate=row2.close) + current_profit_btc = trade.calc_profit(rate=row2.close) lock_pair_until = row2.Index trades.append( ( pair, current_profit_percent, - current_profit_BTC, + current_profit_btc, row2.Index - row.Index ) ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index dcdafc946..686f15a17 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -118,8 +118,7 @@ def optimizer(params): backtesting.populate_buy_trend = buy_strategy_generator(params) results = backtest(OPTIMIZE_CONFIG['stake_amount'], PROCESSED) - - result = format_results(results) + result_explanation = format_results(results) total_profit = results.profit_percent.sum() trade_count = len(results.index) @@ -135,20 +134,17 @@ def optimizer(params): loss = trade_loss + profit_loss _CURRENT_TRIES += 1 - result_data = { + log_results({ 'loss': loss, 'current_tries': _CURRENT_TRIES, 'total_tries': TOTAL_TRIES, - 'result': result, - } - log_results(result_data) + 'result': result_explanation, + }) return { 'loss': loss, 'status': STATUS_OK, - 'result': result, - 'total_profit': total_profit, - 'avg_profit': results.profit_percent.mean() * 100.0, + 'result': result_explanation, } From 7f44ba6df48579659d379d2f7614d67fb91f221d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 10:08:10 +0200 Subject: [PATCH 4/5] unit tests for optimize.hyperopt --- freqtrade/optimize/hyperopt.py | 14 ++-- freqtrade/tests/optimize/test_hyperopt.py | 81 +++++++++++++++++++++-- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 686f15a17..aeb7c1456 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -25,12 +25,10 @@ logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) logger = logging.getLogger(__name__) - # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 TOTAL_TRIES = None _CURRENT_TRIES = 0 - CURRENT_BEST_LOSS = 100 # this is expexted avg profit * expected trade count @@ -111,6 +109,13 @@ def log_results(results): sys.stdout.flush() +def calculate_loss(total_profit: float, trade_count: int): + """ objective function, returns smaller number for more optimal results """ + trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) + profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) + return trade_loss + profit_loss + + def optimizer(params): global _CURRENT_TRIES @@ -129,9 +134,8 @@ def optimizer(params): 'loss': float('inf') } - trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) - profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - loss = trade_loss + profit_loss + loss = calculate_loss(total_profit, trade_count) + _CURRENT_TRIES += 1 log_results({ diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a8bfe7dd4..104b3bfdd 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,6 +1,79 @@ -# pragma pylint: disable=missing-docstring,W0212 +# pragma pylint: disable=missing-docstring,W0212,C0103 + +from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ + log_results -def test_optimizer(default_conf, mocker): - # TODO: implement test - pass +def test_loss_calculation_prefer_correct_trade_count(): + correct = calculate_loss(1, TARGET_TRADES) + over = calculate_loss(1, TARGET_TRADES + 100) + under = calculate_loss(1, TARGET_TRADES - 100) + assert over > correct + assert under > correct + + +def test_loss_calculation_has_limited_profit(): + correct = calculate_loss(EXPECTED_MAX_PROFIT, TARGET_TRADES) + over = calculate_loss(EXPECTED_MAX_PROFIT * 2, TARGET_TRADES) + under = calculate_loss(EXPECTED_MAX_PROFIT / 2, TARGET_TRADES) + assert over == correct + assert under > correct + + +def create_trials(mocker): + return mocker.Mock( + results=[{ + 'loss': 1, + 'result': 'foo' + }] + ) + + +def test_start_calls_fmin(mocker): + mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker)) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + + args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False) + start(args) + + mock_fmin.assert_called_once() + + +def test_start_uses_mongotrials(mocker): + mock_mongotrials = mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', + return_value=create_trials(mocker)) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + + args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True) + start(args) + + mock_mongotrials.assert_called_once() + + +def test_log_results_if_loss_improves(mocker): + logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') + global CURRENT_BEST_LOSS + CURRENT_BEST_LOSS = 2 + log_results({ + 'loss': 1, + 'current_tries': 1, + 'total_tries': 2, + 'result': 'foo' + }) + + logger.assert_called_once() + + +def test_no_log_if_loss_does_not_improve(mocker): + logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') + global CURRENT_BEST_LOSS + CURRENT_BEST_LOSS = 2 + log_results({ + 'loss': 3, + }) + + assert not logger.called From a36fd00f6a76e54da1135b354f52aa39e4f93608 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 18:39:23 +0200 Subject: [PATCH 5/5] also print dot when hyperopt eval result is fail --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index aeb7c1456..d0d0916f8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -129,6 +129,7 @@ def optimizer(params): trade_count = len(results.index) if trade_count == 0: + print('.', end='') return { 'status': STATUS_FAIL, 'loss': float('inf')