import re from datetime import timedelta from pathlib import Path import joblib import pandas as pd import pytest from arrow import Arrow from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN from freqtrade.data import history from freqtrade.data.btanalysis import (get_latest_backtest_filename, load_backtest_data, load_backtest_stats) from freqtrade.edge import PairInfo from freqtrade.enums import ExitType from freqtrade.optimize.optimize_reports import (_get_resample_from_period, generate_backtest_stats, generate_daily_stats, generate_edge_table, generate_exit_reason_stats, generate_pair_metrics, generate_periodic_breakdown_stats, generate_strategy_comparison, generate_trading_stats, show_sorted_pairlist, store_backtest_signal_candles, store_backtest_stats, text_table_bt_results, text_table_exit_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver from tests.conftest import CURRENT_TEST_STRATEGY from tests.data.test_history import _backup_file, _clean_test_file def test_text_table_bt_results(): results = pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], 'profit_ratio': [0.1, 0.2, -0.05], 'profit_abs': [0.2, 0.4, -0.1], 'trade_duration': [10, 30, 20], } ) result_str = ( '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |' ' Avg Duration | Win Draw Loss Win% |\n' '|---------+--------+----------------+----------------+------------------+----------------+' '----------------+-------------------------|\n' '| ETH/BTC | 3 | 8.33 | 25.00 | 0.50000000 | 12.50 |' ' 0:20:00 | 2 0 1 66.7 |\n' '| TOTAL | 3 | 8.33 | 25.00 | 0.50000000 | 12.50 |' ' 0:20:00 | 2 0 1 66.7 |' ) pair_results = generate_pair_metrics(['ETH/BTC'], stake_currency='BTC', starting_balance=4, results=results) assert text_table_bt_results(pair_results, stake_currency='BTC') == result_str def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): default_conf.update({'strategy': CURRENT_TEST_STRATEGY}) StrategyResolver.load_strategy(default_conf) results = {'DefStrat': { 'results': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, Arrow(2017, 11, 14, 21, 36, 00).datetime, Arrow(2017, 11, 14, 22, 12, 00).datetime, Arrow(2017, 11, 14, 22, 44, 00).datetime], "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, Arrow(2017, 11, 14, 22, 10, 00).datetime, Arrow(2017, 11, 14, 22, 43, 00).datetime, Arrow(2017, 11, 14, 22, 58, 00).datetime], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], "is_open": [False, False, False, True], "is_short": [False, False, False, False], "stake_amount": [0.01, 0.01, 0.01, 0.01], "exit_reason": [ExitType.ROI, ExitType.STOP_LOSS, ExitType.ROI, ExitType.FORCE_EXIT] }), 'config': default_conf, 'locks': [], 'final_balance': 1000.02, 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, 'canceled_entry_orders': 0, 'replaced_entry_orders': 0, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, 'run_id': '123', } } timerange = TimeRange.parse_timerange('1510688220-1510700340') min_date = Arrow.fromtimestamp(1510688220) max_date = Arrow.fromtimestamp(1510700340) btdata = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) stats = generate_backtest_stats(btdata, results, min_date, max_date) assert isinstance(stats, dict) assert 'strategy' in stats assert 'DefStrat' in stats['strategy'] assert 'strategy_comparison' in stats strat_stats = stats['strategy']['DefStrat'] assert strat_stats['backtest_start'] == min_date.strftime(DATETIME_PRINT_FORMAT) assert strat_stats['backtest_end'] == max_date.strftime(DATETIME_PRINT_FORMAT) assert strat_stats['total_trades'] == len(results['DefStrat']['results']) # Above sample had no loosing trade assert strat_stats['max_drawdown_account'] == 0.0 # Retry with losing trade results = {'DefStrat': { 'results': pd.DataFrame( {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, -0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003], "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, Arrow(2017, 11, 14, 21, 36, 00).datetime, Arrow(2017, 11, 14, 22, 12, 00).datetime, Arrow(2017, 11, 14, 22, 44, 00).datetime], "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, Arrow(2017, 11, 14, 22, 10, 00).datetime, Arrow(2017, 11, 14, 22, 43, 00).datetime, Arrow(2017, 11, 14, 22, 58, 00).datetime], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217], "trade_duration": [123, 34, 31, 14], "is_open": [False, False, False, True], "is_short": [False, False, False, False], "stake_amount": [0.01, 0.01, 0.01, 0.01], "exit_reason": [ExitType.ROI, ExitType.ROI, ExitType.STOP_LOSS, ExitType.FORCE_EXIT] }), 'config': default_conf, 'locks': [], 'final_balance': 1000.02, 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, 'canceled_entry_orders': 0, 'replaced_entry_orders': 0, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, 'run_id': '124', } } stats = generate_backtest_stats(btdata, results, min_date, max_date) assert isinstance(stats, dict) assert 'strategy' in stats assert 'DefStrat' in stats['strategy'] assert 'strategy_comparison' in stats strat_stats = stats['strategy']['DefStrat'] assert pytest.approx(strat_stats['max_drawdown_account']) == 1.399999e-08 assert strat_stats['drawdown_start'] == '2017-11-14 22:10:00' assert strat_stats['drawdown_end'] == '2017-11-14 22:43:00' assert strat_stats['drawdown_end_ts'] == 1510699380000 assert strat_stats['drawdown_start_ts'] == 1510697400000 assert strat_stats['pairlist'] == ['UNITTEST/BTC'] # Test storing stats filename = Path(tmpdir / 'btresult.json') filename_last = Path(tmpdir / LAST_BT_RESULT_FN) _backup_file(filename_last, copy_file=True) assert not filename.is_file() store_backtest_stats(filename, stats, '2022_01_01_15_05_13') # get real Filename (it's btresult-.json) last_fn = get_latest_backtest_filename(filename_last.parent) assert re.match(r"btresult-.*\.json", last_fn) filename1 = Path(tmpdir / last_fn) assert filename1.is_file() content = filename1.read_text() assert 'max_drawdown_account' in content assert 'strategy' in content assert 'pairlist' in content assert filename_last.is_file() _clean_test_file(filename_last) filename1.unlink() def test_store_backtest_stats(testdatadir, mocker): dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_json') store_backtest_stats(testdatadir, {'metadata': {}}, '2022_01_01_15_05_13') assert dump_mock.call_count == 3 assert isinstance(dump_mock.call_args_list[0][0][0], Path) assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'backtest-result')) dump_mock.reset_mock() filename = testdatadir / 'testresult.json' store_backtest_stats(filename, {'metadata': {}}, '2022_01_01_15_05_13') assert dump_mock.call_count == 3 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-.json assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult')) def test_store_backtest_candles(testdatadir, mocker): dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_joblib') candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} # mock directory exporting store_backtest_signal_candles(testdatadir, candle_dict, '2022_01_01_15_05_13') assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) dump_mock.reset_mock() # mock file exporting filename = Path(testdatadir / 'testresult') store_backtest_signal_candles(filename, candle_dict, '2022_01_01_15_05_13') assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-_signals.pkl assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) dump_mock.reset_mock() def test_write_read_backtest_candles(tmpdir): candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} # test directory exporting stored_file = store_backtest_signal_candles(Path(tmpdir), candle_dict, '2022_01_01_15_05_13') scp = open(stored_file, "rb") pickled_signal_candles = joblib.load(scp) scp.close() assert pickled_signal_candles.keys() == candle_dict.keys() assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) _clean_test_file(stored_file) # test file exporting filename = Path(tmpdir / 'testresult') stored_file = store_backtest_signal_candles(filename, candle_dict, '2022_01_01_15_05_13') scp = open(stored_file, "rb") pickled_signal_candles = joblib.load(scp) scp.close() assert pickled_signal_candles.keys() == candle_dict.keys() assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) _clean_test_file(stored_file) def test_generate_pair_metrics(): results = pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC'], 'profit_ratio': [0.1, 0.2], 'profit_abs': [0.2, 0.4], 'trade_duration': [10, 30], 'wins': [2, 0], 'draws': [0, 0], 'losses': [0, 0] } ) pair_results = generate_pair_metrics(['ETH/BTC'], stake_currency='BTC', starting_balance=2, results=results) assert isinstance(pair_results, list) assert len(pair_results) == 2 assert pair_results[-1]['key'] == 'TOTAL' assert ( pytest.approx(pair_results[-1]['profit_mean_pct']) == pair_results[-1]['profit_mean'] * 100) assert ( pytest.approx(pair_results[-1]['profit_sum_pct']) == pair_results[-1]['profit_sum'] * 100) def test_generate_daily_stats(testdatadir): filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) res = generate_daily_stats(bt_data) assert isinstance(res, dict) assert round(res['backtest_best_day'], 4) == 0.1796 assert round(res['backtest_worst_day'], 4) == -0.1468 assert res['winning_days'] == 19 assert res['draw_days'] == 0 assert res['losing_days'] == 2 # Select empty dataframe! res = generate_daily_stats(bt_data.loc[bt_data['open_date'] == '2000-01-01', :]) assert isinstance(res, dict) assert round(res['backtest_best_day'], 4) == 0.0 assert res['winning_days'] == 0 assert res['draw_days'] == 0 assert res['losing_days'] == 0 def test_generate_trading_stats(testdatadir): filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) res = generate_trading_stats(bt_data) assert isinstance(res, dict) assert res['winner_holding_avg'] == timedelta(seconds=1440) assert res['loser_holding_avg'] == timedelta(days=1, seconds=21420) assert 'wins' in res assert 'losses' in res assert 'draws' in res # Select empty dataframe! res = generate_trading_stats(bt_data.loc[bt_data['open_date'] == '2000-01-01', :]) assert res['wins'] == 0 assert res['losses'] == 0 def test_text_table_exit_reason(): results = pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], 'profit_ratio': [0.1, 0.2, -0.1], 'profit_abs': [0.2, 0.4, -0.2], 'trade_duration': [10, 30, 10], 'wins': [2, 0, 0], 'draws': [0, 0, 0], 'losses': [0, 0, 1], 'exit_reason': [ExitType.ROI, ExitType.ROI, ExitType.STOP_LOSS] } ) result_str = ( '| Exit Reason | Exits | Win Draws Loss Win% | Avg Profit % | Cum Profit % |' ' Tot Profit BTC | Tot Profit % |\n' '|---------------+---------+--------------------------+----------------+----------------+' '------------------+----------------|\n' '| roi | 2 | 2 0 0 100 | 15 | 30 |' ' 0.6 | 15 |\n' '| stop_loss | 1 | 0 0 1 0 | -10 | -10 |' ' -0.2 | -5 |' ) exit_reason_stats = generate_exit_reason_stats(max_open_trades=2, results=results) assert text_table_exit_reason(exit_reason_stats=exit_reason_stats, stake_currency='BTC') == result_str def test_generate_sell_reason_stats(): results = pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], 'profit_ratio': [0.1, 0.2, -0.1], 'profit_abs': [0.2, 0.4, -0.2], 'trade_duration': [10, 30, 10], 'wins': [2, 0, 0], 'draws': [0, 0, 0], 'losses': [0, 0, 1], 'exit_reason': [ExitType.ROI.value, ExitType.ROI.value, ExitType.STOP_LOSS.value] } ) exit_reason_stats = generate_exit_reason_stats(max_open_trades=2, results=results) roi_result = exit_reason_stats[0] assert roi_result['exit_reason'] == 'roi' assert roi_result['trades'] == 2 assert pytest.approx(roi_result['profit_mean']) == 0.15 assert roi_result['profit_mean_pct'] == round(roi_result['profit_mean'] * 100, 2) assert pytest.approx(roi_result['profit_mean']) == 0.15 assert roi_result['profit_mean_pct'] == round(roi_result['profit_mean'] * 100, 2) stop_result = exit_reason_stats[1] assert stop_result['exit_reason'] == 'stop_loss' assert stop_result['trades'] == 1 assert pytest.approx(stop_result['profit_mean']) == -0.1 assert stop_result['profit_mean_pct'] == round(stop_result['profit_mean'] * 100, 2) assert pytest.approx(stop_result['profit_mean']) == -0.1 assert stop_result['profit_mean_pct'] == round(stop_result['profit_mean'] * 100, 2) def test_text_table_strategy(testdatadir): filename = testdatadir / "backtest_results/backtest-result_multistrat.json" bt_res_data = load_backtest_stats(filename) bt_res_data_comparison = bt_res_data.pop('strategy_comparison') result_str = ( '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' ' Tot Profit % | Avg Duration | Win Draw Loss Win% | Drawdown |\n' '|----------------+--------+----------------+----------------+------------------+' '----------------+----------------+-------------------------+-----------------------|\n' '| StrategyTestV2 | 179 | 0.08 | 14.39 | 0.02608550 |' ' 260.85 | 3:40:00 | 170 0 9 95.0 | 0.00308222 BTC 8.67% |\n' '| TestStrategy | 179 | 0.08 | 14.39 | 0.02608550 |' ' 260.85 | 3:40:00 | 170 0 9 95.0 | 0.00308222 BTC 8.67% |' ) strategy_results = generate_strategy_comparison(bt_stats=bt_res_data['strategy']) assert strategy_results == bt_res_data_comparison assert text_table_strategy(strategy_results, 'BTC') == result_str def test_generate_edge_table(): results = {} results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) assert generate_edge_table(results).count('+') == 7 assert generate_edge_table(results).count('| ETH/BTC |') == 1 assert generate_edge_table(results).count( '| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1 def test_generate_periodic_breakdown_stats(testdatadir): filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename).to_dict(orient='records') res = generate_periodic_breakdown_stats(bt_data, 'day') assert isinstance(res, list) assert len(res) == 21 day = res[0] assert 'date' in day assert 'draws' in day assert 'loses' in day assert 'wins' in day assert 'profit_abs' in day # Select empty dataframe! res = generate_periodic_breakdown_stats([], 'day') assert res == [] def test__get_resample_from_period(): assert _get_resample_from_period('day') == '1d' assert _get_resample_from_period('week') == '1w' assert _get_resample_from_period('month') == '1M' with pytest.raises(ValueError, match=r"Period noooo is not supported."): _get_resample_from_period('noooo') def test_show_sorted_pairlist(testdatadir, default_conf, capsys): filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_stats(filename) default_conf['backtest_show_pair_list'] = True show_sorted_pairlist(default_conf, bt_data) out, err = capsys.readouterr() assert 'Pairs for Strategy StrategyTestV3: \n[' in out assert 'TOTAL' not in out assert '"ETH/BTC", // ' in out