Merge pull request #6177 from freqtrade/remove_old_bt_format

Remove old bt format

freqtrade/data/btanalysis.py

@@ -9,16 +9,13 @@ import numpy as np
 import pandas as pd
 
 from freqtrade.constants import LAST_BT_RESULT_FN
+from freqtrade.exceptions import OperationalException
 from freqtrade.misc import json_load
 from freqtrade.persistence import LocalTrade, Trade, init_db
 
 
 logger = logging.getLogger(__name__)
 
-# Old format - maybe remove?
-BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
-                       "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
-
 # Newest format
 BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
                    'open_rate', 'close_rate',
@@ -162,23 +159,9 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                               )
     else:
         # old format - only with lists.
-        df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
-        if not df.empty:
-            df['open_date'] = pd.to_datetime(df['open_date'],
-                                             unit='s',
-                                             utc=True,
-                                             infer_datetime_format=True
-                                             )
-            df['close_date'] = pd.to_datetime(df['close_date'],
-                                              unit='s',
-                                              utc=True,
-                                              infer_datetime_format=True
-                                              )
-            # Create compatibility with new format
-            df['profit_abs'] = df['close_rate'] - df['open_rate']
+        raise OperationalException(
+            "Backtest-results with only trades data are no longer supported.")
     if not df.empty:
-        if 'profit_ratio' not in df.columns:
-            df['profit_ratio'] = df['profit_percent']
         df = df.sort_values("open_date").reset_index(drop=True)
     return df
 

tests/data/test_btanalysis.py

@@ -8,14 +8,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import LAST_BT_RESULT_FN
-from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, BT_DATA_COLUMNS_OLD,
-                                       analyze_trade_parallelism, calculate_csum,
+from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_csum,
                                        calculate_market_change, calculate_max_drawdown,
                                        calculate_underwater, combine_dataframes_with_mean,
                                        create_cum_profit, extract_trades_of_period,
                                        get_latest_backtest_filename, get_latest_hyperopt_file,
                                        load_backtest_data, load_trades, load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
+from freqtrade.exceptions import OperationalException
 from tests.conftest import create_mock_trades
 from tests.conftest_trades import MOCK_TRADE_COUNT
 
@@ -51,20 +51,14 @@ def test_get_latest_hyperopt_file(testdatadir, mocker):
     assert res == testdatadir.parent / "hyperopt_results.pickle"
 
 
-def test_load_backtest_data_old_format(testdatadir):
+def test_load_backtest_data_old_format(testdatadir, mocker):
 
-    filename = testdatadir / "backtest-result_test.json"
-    bt_data = load_backtest_data(filename)
-    assert isinstance(bt_data, DataFrame)
-    assert list(bt_data.columns) == BT_DATA_COLUMNS_OLD + ['profit_abs', 'profit_ratio']
-    assert len(bt_data) == 179
+    filename = testdatadir / "backtest-result_test222.json"
+    mocker.patch('freqtrade.data.btanalysis.load_backtest_stats', return_value=[])
 
-    # Test loading from string (must yield same result)
-    bt_data2 = load_backtest_data(str(filename))
-    assert bt_data.equals(bt_data2)
-
-    with pytest.raises(ValueError, match=r"File .* does not exist\."):
-        load_backtest_data(str("filename") + "nofile")
+    with pytest.raises(OperationalException,
+                       match=r"Backtest-results with only trades data are no longer supported."):
+        load_backtest_data(filename)
 
 
 def test_load_backtest_data_new_format(testdatadir):
@@ -167,8 +161,8 @@ def test_extract_trades_of_period(testdatadir):
     assert trades1.iloc[-1].close_date == Arrow(2017, 11, 14, 15, 25, 0).datetime
 
 
-def test_analyze_trade_parallelism(default_conf, mocker, testdatadir):
-    filename = testdatadir / "backtest-result_test.json"
+def test_analyze_trade_parallelism(testdatadir):
+    filename = testdatadir / "backtest-result_new.json"
     bt_data = load_backtest_data(filename)
 
     res = analyze_trade_parallelism(bt_data, "5m")
@@ -242,7 +236,7 @@ def test_combine_dataframes_with_mean_no_data(testdatadir):
 
 
 def test_create_cum_profit(testdatadir):
-    filename = testdatadir / "backtest-result_test.json"
+    filename = testdatadir / "backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
@@ -258,7 +252,7 @@ def test_create_cum_profit(testdatadir):
 
 
 def test_create_cum_profit1(testdatadir):
-    filename = testdatadir / "backtest-result_test.json"
+    filename = testdatadir / "backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     # Move close-time to "off" the candle, to make sure the logic still works
     bt_data.loc[:, 'close_date'] = bt_data.loc[:, 'close_date'] + DateOffset(seconds=20)
@@ -304,7 +298,7 @@ def test_calculate_max_drawdown(testdatadir):
 
 
 def test_calculate_csum(testdatadir):
-    filename = testdatadir / "backtest-result_test.json"
+    filename = testdatadir / "backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     csum_min, csum_max = calculate_csum(bt_data)
 

tests/test_plotting.py

@@ -45,7 +45,7 @@ def test_init_plotscript(default_conf, mocker, testdatadir):
     default_conf['trade_source'] = "file"
     default_conf['timeframe'] = "5m"
     default_conf["datadir"] = testdatadir
-    default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
+    default_conf['exportfilename'] = testdatadir / "backtest-result_new.json"
     supported_markets = ["TRX/BTC", "ADA/BTC"]
     ret = init_plotscript(default_conf, supported_markets)
     assert "ohlcv" in ret
@@ -157,7 +157,7 @@ def test_plot_trades(testdatadir, caplog):
     assert fig == fig1
     assert log_has("No trades found.", caplog)
     pair = "ADA/BTC"
-    filename = testdatadir / "backtest-result_test.json"
+    filename = testdatadir / "backtest-result_new.json"
     trades = load_backtest_data(filename)
     trades = trades.loc[trades['pair'] == pair]
 
@@ -294,7 +294,7 @@ def test_generate_plot_file(mocker, caplog):
 
 
 def test_add_profit(testdatadir):
-    filename = testdatadir / "backtest-result_test.json"
+    filename = testdatadir / "backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
@@ -314,7 +314,7 @@ def test_add_profit(testdatadir):
 
 
 def test_generate_profit_graph(testdatadir):
-    filename = testdatadir / "backtest-result_test.json"
+    filename = testdatadir / "backtest-result_new.json"
     trades = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
     pairs = ["TRX/BTC", "XLM/BTC"]
@@ -381,7 +381,7 @@ def test_load_and_plot_trades(default_conf, mocker, caplog, testdatadir):
 
     default_conf['trade_source'] = 'file'
     default_conf["datadir"] = testdatadir
-    default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
+    default_conf['exportfilename'] = testdatadir / "backtest-result_new.json"
     default_conf['indicators1'] = ["sma5", "ema10"]
     default_conf['indicators2'] = ["macd"]
     default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"]
@@ -452,7 +452,7 @@ def test_plot_profit(default_conf, mocker, testdatadir):
                        match=r"No trades found, cannot generate Profit-plot.*"):
         plot_profit(default_conf)
 
-    default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
+    default_conf['exportfilename'] = testdatadir / "backtest-result_new.json"
 
     plot_profit(default_conf)