Merge pull request #6685 from freqtrade/bt_load_history

Backtesting load history

freqtrade/data/btanalysis.py

@@ -149,7 +149,14 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
     return data
 
 
-def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]):
+def load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]):
+    """
+    Load one strategy from multi-strategy result
+    and merge it with results
+    :param strategy_name: Name of the strategy contained in the result
+    :param filename: Backtest-result-filename to load
+    :param results: dict to merge the result to.
+    """
     bt_data = load_backtest_stats(filename)
     for k in ('metadata', 'strategy'):
         results[k][strategy_name] = bt_data[k][strategy_name]
@@ -160,6 +167,30 @@ def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results:
             break
 
 
+def _get_backtest_files(dirname: Path) -> List[Path]:
+    return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))))
+
+
+def get_backtest_resultlist(dirname: Path):
+    """
+    Get list of backtest results read from metadata files
+    """
+    results = []
+    for filename in _get_backtest_files(dirname):
+        metadata = load_backtest_metadata(filename)
+        if not metadata:
+            continue
+        for s, v in metadata.items():
+            results.append({
+                'filename': filename.name,
+                'strategy': s,
+                'run_id': v['run_id'],
+                'backtest_start_time': v['backtest_start_time'],
+
+            })
+    return results
+
+
 def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str],
                                  min_backtest_date: datetime = None) -> Dict[str, Any]:
     """
@@ -179,7 +210,7 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s
     }
 
     # Weird glob expression here avoids including .meta.json files.
-    for filename in reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))):
+    for filename in _get_backtest_files(dirname):
         metadata = load_backtest_metadata(filename)
         if not metadata:
             # Files are sorted from newest to oldest. When file without metadata is encountered it
@@ -202,7 +233,7 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s
 
             if strategy_metadata['run_id'] == run_id:
                 del run_ids[strategy_name]
-                _load_and_merge_backtest_result(strategy_name, filename, results)
+                load_and_merge_backtest_result(strategy_name, filename, results)
 
         if len(run_ids) == 0:
             break

freqtrade/rpc/api_server/api_backtest.py

@@ -1,13 +1,16 @@
 import asyncio
 import logging
 from copy import deepcopy
+from typing import Any, Dict, List
 
 from fastapi import APIRouter, BackgroundTasks, Depends
 
 from freqtrade.configuration.config_validation import validate_config_consistency
+from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result
 from freqtrade.enums import BacktestState
 from freqtrade.exceptions import DependencyException
-from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse
+from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest,
+                                                  BacktestResponse)
 from freqtrade.rpc.api_server.deps import get_config, is_webserver_mode
 from freqtrade.rpc.api_server.webserver import ApiServer
 from freqtrade.rpc.rpc import RPCException
@@ -200,3 +203,30 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)):
         "progress": 0,
         "status_msg": "Backtest ended",
     }
+
+
+@router.get('/backtest/history', response_model=List[BacktestHistoryEntry], tags=['webserver', 'backtest'])
+def api_backtest_history(config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
+    # Get backtest result history, read from metadata files
+    return get_backtest_resultlist(config['user_data_dir'] / 'backtest_results')
+
+
+@router.get('/backtest/history/result', response_model=BacktestResponse, tags=['webserver', 'backtest'])
+def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
+    # Get backtest result history, read from metadata files
+    fn = config['user_data_dir'] / 'backtest_results' / filename
+    results: Dict[str, Any] = {
+        'metadata': {},
+        'strategy': {},
+        'strategy_comparison': [],
+    }
+
+    load_and_merge_backtest_result(strategy, fn, results)
+    return {
+        "status": "ended",
+        "running": False,
+        "step": "",
+        "progress": 1,
+        "status_msg": "Historic result",
+        "backtest_result": results,
+    }

freqtrade/rpc/api_server/api_schemas.py

@@ -421,6 +421,13 @@ class BacktestResponse(BaseModel):
     backtest_result: Optional[Dict[str, Any]]
 
 
+class BacktestHistoryEntry(BaseModel):
+    filename: str
+    strategy: str
+    run_id: str
+    backtest_start_time: int
+
+
 class SysInfo(BaseModel):
     cpu_pct: List[float]
     ram_pct: float

freqtrade/rpc/api_server/api_v1.py

@@ -35,7 +35,8 @@ logger = logging.getLogger(__name__)
 # 1.13: forcebuy supports stake_amount
 # versions 2.xx -> futures/short branch
 # 2.14: Add entry/exit orders to trade response
-API_VERSION = 2.14
+# 2.15: Add backtest history endpoints
+API_VERSION = 2.15
 
 # Public API, requires no auth.
 router_public = APIRouter()

tests/commands/test_commands.py

@@ -1429,7 +1429,7 @@ def test_backtesting_show(mocker, testdatadir, capsys):
     args = [
         "backtesting-show",
         "--export-filename",
-        f"{testdatadir / 'backtest-result_new.json'}",
+        f"{testdatadir / 'backtest_results/backtest-result_new.json'}",
         "--show-pair-list"
     ]
     pargs = get_args(args)

tests/data/test_btanalysis.py

@@ -27,18 +27,19 @@ def test_get_latest_backtest_filename(testdatadir, mocker):
 
     with pytest.raises(ValueError,
                        match=r"Directory .* does not seem to contain .*"):
-        get_latest_backtest_filename(testdatadir.parent)
+        get_latest_backtest_filename(testdatadir)
 
-    res = get_latest_backtest_filename(testdatadir)
+    testdir_bt = testdatadir / "backtest_results"
+    res = get_latest_backtest_filename(testdir_bt)
     assert res == 'backtest-result_new.json'
 
-    res = get_latest_backtest_filename(str(testdatadir))
+    res = get_latest_backtest_filename(str(testdir_bt))
     assert res == 'backtest-result_new.json'
 
     mocker.patch("freqtrade.data.btanalysis.json_load", return_value={})
 
     with pytest.raises(ValueError, match=r"Invalid '.last_result.json' format."):
-        get_latest_backtest_filename(testdatadir)
+        get_latest_backtest_filename(testdir_bt)
 
 
 def test_get_latest_hyperopt_file(testdatadir):
@@ -81,7 +82,7 @@ def test_load_backtest_data_old_format(testdatadir, mocker):
 
 def test_load_backtest_data_new_format(testdatadir):
 
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     assert isinstance(bt_data, DataFrame)
     assert set(bt_data.columns) == set(BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp'])
@@ -92,19 +93,19 @@ def test_load_backtest_data_new_format(testdatadir):
     assert bt_data.equals(bt_data2)
 
     # Test loading from folder (must yield same result)
-    bt_data3 = load_backtest_data(testdatadir)
+    bt_data3 = load_backtest_data(testdatadir / "backtest_results")
     assert bt_data.equals(bt_data3)
 
     with pytest.raises(ValueError, match=r"File .* does not exist\."):
         load_backtest_data(str("filename") + "nofile")
 
     with pytest.raises(ValueError, match=r"Unknown dataformat."):
-        load_backtest_data(testdatadir / LAST_BT_RESULT_FN)
+        load_backtest_data(testdatadir / "backtest_results" / LAST_BT_RESULT_FN)
 
 
 def test_load_backtest_data_multi(testdatadir):
 
-    filename = testdatadir / "backtest-result_multistrat.json"
+    filename = testdatadir / "backtest_results/backtest-result_multistrat.json"
     for strategy in ('StrategyTestV2', 'TestStrategy'):
         bt_data = load_backtest_data(filename, strategy=strategy)
         assert isinstance(bt_data, DataFrame)
@@ -182,7 +183,7 @@ def test_extract_trades_of_period(testdatadir):
 
 
 def test_analyze_trade_parallelism(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
 
     res = analyze_trade_parallelism(bt_data, "5m")
@@ -256,7 +257,7 @@ def test_combine_dataframes_with_mean_no_data(testdatadir):
 
 
 def test_create_cum_profit(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
@@ -272,7 +273,7 @@ def test_create_cum_profit(testdatadir):
 
 
 def test_create_cum_profit1(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     # Move close-time to "off" the candle, to make sure the logic still works
     bt_data.loc[:, 'close_date'] = bt_data.loc[:, 'close_date'] + DateOffset(seconds=20)
@@ -294,7 +295,7 @@ def test_create_cum_profit1(testdatadir):
 
 
 def test_calculate_max_drawdown(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     _, hdate, lowdate, hval, lval, drawdown = calculate_max_drawdown(
         bt_data, value_col="profit_abs")
@@ -318,7 +319,7 @@ def test_calculate_max_drawdown(testdatadir):
 
 
 def test_calculate_csum(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     csum_min, csum_max = calculate_csum(bt_data)
 

tests/optimize/test_optimize_reports.py

@@ -228,7 +228,7 @@ def test_generate_pair_metrics():
 
 def test_generate_daily_stats(testdatadir):
 
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     res = generate_daily_stats(bt_data)
     assert isinstance(res, dict)
@@ -248,7 +248,7 @@ def test_generate_daily_stats(testdatadir):
 
 
 def test_generate_trading_stats(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     res = generate_trading_stats(bt_data)
     assert isinstance(res, dict)
@@ -332,7 +332,7 @@ def test_generate_sell_reason_stats():
 
 
 def test_text_table_strategy(testdatadir):
-    filename = testdatadir / "backtest-result_multistrat.json"
+    filename = testdatadir / "backtest_results/backtest-result_multistrat.json"
     bt_res_data = load_backtest_stats(filename)
 
     bt_res_data_comparison = bt_res_data.pop('strategy_comparison')
@@ -364,7 +364,7 @@ def test_generate_edge_table():
 
 
 def test_generate_periodic_breakdown_stats(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename).to_dict(orient='records')
 
     res = generate_periodic_breakdown_stats(bt_data, 'day')
@@ -392,7 +392,7 @@ def test__get_resample_from_period():
 
 
 def test_show_sorted_pairlist(testdatadir, default_conf, capsys):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_stats(filename)
     default_conf['backtest_show_pair_list'] = True
 

tests/rpc/test_rpc_apiserver.py

@@ -1581,6 +1581,38 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
     assert result['status_msg'] == 'Backtest reset'
 
 
+def test_api_backtest_history(botclient, mocker, testdatadir):
+    ftbot, client = botclient
+    mocker.patch('freqtrade.data.btanalysis._get_backtest_files',
+                 return_value=[
+                     testdatadir / 'backtest_results/backtest-result_multistrat.json',
+                     testdatadir / 'backtest_results/backtest-result_new.json'
+                     ])
+
+    rc = client_get(client, f"{BASE_URI}/backtest/history")
+    assert_response(rc, 502)
+    ftbot.config['user_data_dir'] = testdatadir
+    ftbot.config['runmode'] = RunMode.WEBSERVER
+
+    rc = client_get(client, f"{BASE_URI}/backtest/history")
+    assert_response(rc)
+    result = rc.json()
+    assert len(result) == 3
+    fn = result[0]['filename']
+    assert fn == "backtest-result_multistrat.json"
+    strategy = result[0]['strategy']
+    rc = client_get(client, f"{BASE_URI}/backtest/history/result?filename={fn}&strategy={strategy}")
+    assert_response(rc)
+    result2 = rc.json()
+    assert result2
+    assert result2['status'] == 'ended'
+    assert not result2['running']
+    assert result2['progress'] == 1
+    # Only one strategy loaded - even though we use multiresult
+    assert len(result2['backtest_result']['strategy']) == 1
+    assert result2['backtest_result']['strategy'][strategy]
+
+
 def test_health(botclient):
     ftbot, client = botclient
 

tests/test_plotting.py

@@ -157,7 +157,7 @@ def test_plot_trades(testdatadir, caplog):
     assert fig == fig1
     assert log_has("No trades found.", caplog)
     pair = "ADA/BTC"
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     trades = load_backtest_data(filename)
     trades = trades.loc[trades['pair'] == pair]
 
@@ -298,7 +298,7 @@ def test_generate_plot_file(mocker, caplog):
 
 
 def test_add_profit(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
@@ -318,7 +318,7 @@ def test_add_profit(testdatadir):
 
 
 def test_generate_profit_graph(testdatadir):
-    filename = testdatadir / "backtest-result_new.json"
+    filename = testdatadir / "backtest_results/backtest-result_new.json"
     trades = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
     pairs = ["TRX/BTC", "XLM/BTC"]
@@ -456,7 +456,7 @@ def test_plot_profit(default_conf, mocker, testdatadir):
                        match=r"No trades found, cannot generate Profit-plot.*"):
         plot_profit(default_conf)
 
-    default_conf['exportfilename'] = testdatadir / "backtest-result_new.json"
+    default_conf['exportfilename'] = testdatadir / "backtest_results/backtest-result_new.json"
 
     plot_profit(default_conf)
 

tests/testdata/backtest_results/backtest-result_multistrat.meta.json

@@ -0,0 +1,10 @@
+{
+    "StrategyTestV2": {
+        "run_id": "430d0271075ef327edbb23088f4db4ebe51a3dbf",
+        "backtest_start_time": 1648904006
+    },
+    "TestStrategy": {
+        "run_id": "110d0271075ef327edbb23085102b4ebe51a3d55",
+        "backtest_start_time": 1648904006
+    }
+}

tests/testdata/backtest_results/backtest-result_new.meta.json

@@ -0,0 +1,6 @@
+{
+    "StrategyTestV3": {
+        "run_id": "430d0271075ef327edbb23088f4db4ebe51a3dbf",
+        "backtest_start_time": 1648904006
+    }
+}