From b3cb7226467c359b0748cb4866f8f8fcfa649a53 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 20 Apr 2022 13:38:52 +0100 Subject: [PATCH] Use joblib instead of pickle, add signal candle read/write test, move docs to new Advanced Backtesting doc --- docs/advanced-backtesting.md | 75 +++++++++++++++++++++++++ docs/data-analysis.md | 1 + docs/strategy_analysis_example.md | 72 ------------------------ freqtrade/misc.py | 10 ++-- freqtrade/optimize/optimize_reports.py | 8 ++- tests/optimize/test_optimize_reports.py | 48 ++++++++++++++-- 6 files changed, 128 insertions(+), 86 deletions(-) create mode 100644 docs/advanced-backtesting.md diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md new file mode 100644 index 000000000..9a7d71767 --- /dev/null +++ b/docs/advanced-backtesting.md @@ -0,0 +1,75 @@ +# Advanced Backtesting Analysis + +## Analyse the buy/entry and sell/exit tags + +It can be helpful to understand how a strategy behaves according to the buy/entry tags used to +mark up different buy conditions. You might want to see more complex statistics about each buy and +sell condition above those provided by the default backtesting output. You may also want to +determine indicator values on the signal candle that resulted in a trade opening. + +!!! Note + The following buy reason analysis is only available for backtesting, *not hyperopt*. + +We first need to enable the exporting of trades from backtesting: + +```bash +freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades --export-filename=user_data/backtest_results/- +``` + +To analyse the buy tags, we need to use the `freqtrade tag-analysis` command. We need the signal +candles for each opened trade so add the following option to your config file: + +``` +'backtest_signal_candle_export_enable': true, +``` + +This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding +DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy +makes, this file may get quite large, so periodically check your `user_data/backtest_results` +folder to delete old exports. + +Before running your next backtest, make sure you either delete your old backtest results or run +backtesting with the `--cache none` option to make sure no cached results are used. + +If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the +`user_data/backtest_results` folder. + +Now run the buy_reasons.py script, supplying a few options: + +```bash +freqtrade tag-analysis -c -s -t -g0,1,2,3,4 +``` + +The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) +to the most detailed per pair, per buy and per sell tag (4). More options are available by +running with the `-h` option. + +### Tuning the buy tags and sell tags to display + +To show only certain buy and sell tags in the displayed output, use the following two options: + +``` +--buy_reason_list : Comma separated list of buy signals to analyse. Default: "all" +--sell_reason_list : Comma separated list of sell signals to analyse. Default: "stop_loss,trailing_stop_loss" +``` + +For example: + +```bash +freqtrade tag-analysis -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" +``` + +### Outputting signal candle indicators + +The real power of the buy_reasons.py script comes from the ability to print out the indicator +values present on signal candles to allow fine-grained investigation and tuning of buy signal +indicators. To print out a column for a given set of indicators, use the `--indicator-list` +option: + +```bash +freqtrade tag-analysis -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +``` + +The indicators have to be present in your strategy's main DataFrame (either for your main +timeframe or for informatives) otherwise they will simply be ignored in the script +output. diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 9a79ee5ed..926ed3eae 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -122,5 +122,6 @@ Best avoid relative paths, since this starts at the storage location of the jupy * [Strategy debugging](strategy_analysis_example.md) - also available as Jupyter notebook (`user_data/notebooks/strategy_analysis_example.ipynb`) * [Plotting](plotting.md) +* [Tag Analysis](advanced-backtesting.md) Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 48f54c824..ae0c6a6a3 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -250,75 +250,3 @@ fig.show() ``` Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data. - -## Analyse the buy/entry and sell/exit tags - -It can be helpful to understand how a strategy behaves according to the buy/entry tags used to -mark up different buy conditions. You might want to see more complex statistics about each buy and -sell condition above those provided by the default backtesting output. You may also want to -determine indicator values on the signal candle that resulted in a trade opening. - -We first need to enable the exporting of trades from backtesting: - -``` -freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades --export-filename=user_data/backtest_results/- -``` - -To analyse the buy tags, we need to use the buy_reasons.py script in the `scripts/` -folder. We need the signal candles for each opened trade so add the following option to your -config file: - -``` -'backtest_signal_candle_export_enable': true, -``` - -This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding -DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy -makes, this file may get quite large, so periodically check your `user_data/backtest_results` -folder to delete old exports. - -Before running your next backtest, make sure you either delete your old backtest results or run -backtesting with the `--cache none` option to make sure no cached results are used. - -If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the -`user_data/backtest_results` folder. - -Now run the buy_reasons.py script, supplying a few options: - -``` -./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 -``` - -The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) -to the most detailed per pair, per buy and per sell tag (4). More options are available by -running with the `-h` option. - -### Tuning the buy tags and sell tags to display - -To show only certain buy and sell tags in the displayed output, use the following two options: - -``` ---buy_reason_list : Comma separated list of buy signals to analyse. Default: "all" ---sell_reason_list : Comma separated list of sell signals to analyse. Default: "stop_loss,trailing_stop_loss" -``` - -For example: - -``` -./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" -``` - -### Outputting signal candle indicators - -The real power of the buy_reasons.py script comes from the ability to print out the indicator -values present on signal candles to allow fine-grained investigation and tuning of buy signal -indicators. To print out a column for a given set of indicators, use the `--indicator-list` -option: - -``` -./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" -``` - -The indicators have to be present in your strategy's main dataframe (either for your main -timeframe or for informatives) otherwise they will simply be ignored in the script -output. diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 9087ec6e2..be12d8224 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -4,7 +4,6 @@ Various tool function for Freqtrade and scripts import gzip import hashlib import logging -import pickle import re from copy import deepcopy from datetime import datetime @@ -13,6 +12,7 @@ from typing import Any, Iterator, List, Union from typing.io import IO from urllib.parse import urlparse +import joblib import rapidjson from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN @@ -87,7 +87,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = logger.debug(f'done json to "{filename}"') -def file_dump_pickle(filename: Path, data: Any, log: bool = True) -> None: +def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: """ Dump object data into a file :param filename: file to create @@ -96,10 +96,10 @@ def file_dump_pickle(filename: Path, data: Any, log: bool = True) -> None: """ if log: - logger.info(f'dumping pickle to "{filename}"') + logger.info(f'dumping joblib to "{filename}"') with open(filename, 'wb') as fp: - pickle.dump(data, fp) - logger.debug(f'done pickling to "{filename}"') + joblib.dump(data, fp) + logger.debug(f'done joblib dump to "{filename}"') def json_load(datafile: IO) -> Any: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 05eec693e..6288ee16a 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -11,7 +11,7 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_json, file_dump_pickle, +from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, get_backtest_metadata_filename, round_coin_value) @@ -45,7 +45,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) -def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> None: +def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> Path: """ Stores backtest trade signal candles :param recordfilename: Path object, which can either be a filename or a directory. @@ -63,7 +63,9 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict] f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl' ) - file_dump_pickle(filename, candles) + file_dump_joblib(filename, candles) + + return filename def _get_line_floatfmt(stake_currency: str) -> List[str]: diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index d72cf4e86..ff8d420b3 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -2,6 +2,7 @@ import re from datetime import timedelta from pathlib import Path +import joblib import pandas as pd import pytest from arrow import Arrow @@ -204,23 +205,58 @@ def test_store_backtest_stats(testdatadir, mocker): def test_store_backtest_candles(testdatadir, mocker): - dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_pickle') + dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_joblib') - # test directory exporting - store_backtest_signal_candles(testdatadir, {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}) + candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} + + # mock directory exporting + store_backtest_signal_candles(testdatadir, candle_dict) assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) dump_mock.reset_mock() - # test file exporting - filename = testdatadir / 'testresult' - store_backtest_signal_candles(filename, {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}) + # mock file exporting + filename = Path(testdatadir / 'testresult') + store_backtest_signal_candles(filename, candle_dict) assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-_signals.pkl assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) + dump_mock.reset_mock() + + +def test_write_read_backtest_candles(tmpdir): + + candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} + + # test directory exporting + stored_file = store_backtest_signal_candles(Path(tmpdir), candle_dict) + scp = open(stored_file, "rb") + pickled_signal_candles = joblib.load(scp) + scp.close() + + assert pickled_signal_candles.keys() == candle_dict.keys() + assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() + assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ + .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) + + _clean_test_file(stored_file) + + # test file exporting + filename = Path(tmpdir / 'testresult') + stored_file = store_backtest_signal_candles(filename, candle_dict) + scp = open(stored_file, "rb") + pickled_signal_candles = joblib.load(scp) + scp.close() + + assert pickled_signal_candles.keys() == candle_dict.keys() + assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() + assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ + .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) + + _clean_test_file(stored_file) def test_generate_pair_metrics():