diff --git a/docs/backtesting.md b/docs/backtesting.md index 8364d77e4..1efb46b43 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -1,17 +1,19 @@ # Backtesting + This page explains how to validate your strategy performance by using Backtesting. ## Table of Contents + - [Test your strategy with Backtesting](#test-your-strategy-with-backtesting) - [Understand the backtesting result](#understand-the-backtesting-result) ## Test your strategy with Backtesting + Now you have good Buy and Sell strategies, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). - Backtesting will use the crypto-currencies (pair) from your config file and load static tickers located in [/freqtrade/tests/testdata](https://github.com/freqtrade/freqtrade/tree/develop/freqtrade/tests/testdata). @@ -19,70 +21,80 @@ If the 5 min and 1 min ticker for the crypto-currencies to test is not already in the `testdata` folder, backtesting will download them automatically. Testdata files will not be updated until you specify it. -The result of backtesting will confirm you if your bot as more chance to -make a profit than a loss. - +The result of backtesting will confirm you if your bot has better odds of making a profit than a loss. The backtesting is very easy with freqtrade. ### Run a backtesting against the currencies listed in your config file -**With 5 min tickers (Per default)** +#### With 5 min tickers (Per default) + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation ``` -**With 1 min tickers** +#### With 1 min tickers + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --ticker-interval 1m ``` -**Update cached pairs with the latest data** +#### Update cached pairs with the latest data + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-cached ``` -**With live data (do not alter your testdata files)** +#### With live data (do not alter your testdata files) + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --live ``` -**Using a different on-disk ticker-data source** +#### Using a different on-disk ticker-data source + ```bash python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101 ``` -**With a (custom) strategy file** +#### With a (custom) strategy file + ```bash python3 ./freqtrade/main.py -s TestStrategy backtesting ``` + Where `-s TestStrategy` refers to the class name within the strategy file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory -**Exporting trades to file** +#### Exporting trades to file + ```bash python3 ./freqtrade/main.py backtesting --export trades ``` -**Exporting trades to file specifying a custom filename** +#### Exporting trades to file specifying a custom filename + ```bash python3 ./freqtrade/main.py backtesting --export trades --export-filename=backtest_teststrategy.json ``` +#### Running backtest with smaller testset -**Running backtest with smaller testset** Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: + ```bash python3 ./freqtrade/main.py backtesting --timerange=-200 ``` -***Advanced use of timerange*** +#### Advanced use of timerange + Doing `--timerange=-200` will get the last 200 timeframes from your inputdata. You can also specify specific dates, or a range span indexed by start and stop. The full timerange specification: + - Use last 123 tickframes of data: `--timerange=-123` - Use first 123 tickframes of data: `--timerange=123-` - Use tickframes from line 123 through 456: `--timerange=123-456` @@ -92,11 +104,12 @@ The full timerange specification: - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` +#### Downloading new set of ticker data -**Downloading new set of ticker data** To download new set of backtesting ticker data, you can use a download script. If you are using Binance for example: + - create a folder `user_data/data/binance` and copy `pairs.json` in that folder. - update the `pairs.json` to contain the currency pairs you are interested in. @@ -119,33 +132,55 @@ This will download ticker data for all the currency pairs you defined in `pairs. - To download ticker data for only 10 days, use `--days 10`. - Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. - -For help about backtesting usage, please refer to -[Backtesting commands](#backtesting-commands). +For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). ## Understand the backtesting result + The most important in the backtesting is to understand the result. A backtesting result will look like that: + ``` -====================== BACKTESTING REPORT ================================ -pair buy count avg profit % total profit BTC avg duration --------- ----------- -------------- ------------------ -------------- -ETH/BTC 56 -0.67 -0.00075455 62.3 -LTC/BTC 38 -0.48 -0.00036315 57.9 -ETC/BTC 42 -1.15 -0.00096469 67.0 -DASH/BTC 72 -0.62 -0.00089368 39.9 -ZEC/BTC 45 -0.46 -0.00041387 63.2 -XLM/BTC 24 -0.88 -0.00041846 47.7 -NXT/BTC 24 0.68 0.00031833 40.2 -POWR/BTC 35 0.98 0.00064887 45.3 -ADA/BTC 43 -0.39 -0.00032292 55.0 -XMR/BTC 40 -0.40 -0.00032181 47.4 -TOTAL 419 -0.41 -0.00348593 52.9 +======================================== BACKTESTING REPORT ========================================= +| pair | buy count | avg profit % | total profit BTC | avg duration | profit | loss | +|:---------|------------:|---------------:|-------------------:|---------------:|---------:|-------:| +| ETH/BTC | 44 | 0.18 | 0.00159118 | 50.9 | 44 | 0 | +| LTC/BTC | 27 | 0.10 | 0.00051931 | 103.1 | 26 | 1 | +| ETC/BTC | 24 | 0.05 | 0.00022434 | 166.0 | 22 | 2 | +| DASH/BTC | 29 | 0.18 | 0.00103223 | 192.2 | 29 | 0 | +| ZEC/BTC | 65 | -0.02 | -0.00020621 | 202.7 | 62 | 3 | +| XLM/BTC | 35 | 0.02 | 0.00012877 | 242.4 | 32 | 3 | +| BCH/BTC | 12 | 0.62 | 0.00149284 | 50.0 | 12 | 0 | +| POWR/BTC | 21 | 0.26 | 0.00108215 | 134.8 | 21 | 0 | +| ADA/BTC | 54 | -0.19 | -0.00205202 | 191.3 | 47 | 7 | +| XMR/BTC | 24 | -0.43 | -0.00206013 | 120.6 | 20 | 4 | +| TOTAL | 335 | 0.03 | 0.00175246 | 157.9 | 315 | 20 | +2018-06-13 06:57:27,347 - freqtrade.optimize.backtesting - INFO - +====================================== LEFT OPEN TRADES REPORT ====================================== +| pair | buy count | avg profit % | total profit BTC | avg duration | profit | loss | +|:---------|------------:|---------------:|-------------------:|---------------:|---------:|-------:| +| ETH/BTC | 3 | 0.16 | 0.00009619 | 25.0 | 3 | 0 | +| LTC/BTC | 1 | -1.00 | -0.00020118 | 1085.0 | 0 | 1 | +| ETC/BTC | 2 | -1.80 | -0.00071933 | 1092.5 | 0 | 2 | +| DASH/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| ZEC/BTC | 3 | -4.27 | -0.00256826 | 1301.7 | 0 | 3 | +| XLM/BTC | 3 | -1.11 | -0.00066744 | 965.0 | 0 | 3 | +| BCH/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| POWR/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| ADA/BTC | 7 | -3.58 | -0.00503604 | 850.0 | 0 | 7 | +| XMR/BTC | 4 | -3.79 | -0.00303456 | 291.2 | 0 | 4 | +| TOTAL | 23 | -2.63 | -0.01213062 | 750.4 | 3 | 20 | + ``` +The 1st table will contain all trades the bot made. + +The 2nd table will contain all trades the bot had to `forcesell` at the end of the backtest period to prsent a full picture. +These trades are also included in the first table, but are extracted separately for clarity. + The last line will give you the overall performance of your strategy, here: + ``` TOTAL 419 -0.41 -0.00348593 52.9 ``` @@ -161,6 +196,7 @@ strategy, your sell strategy, and also by the `minimal_roi` and As for an example if your minimal_roi is only `"0": 0.01`. You cannot expect the bot to make more profit than 1% (because it will sell every time a trade will reach 1%). + ```json "minimal_roi": { "0": 0.01 @@ -173,6 +209,7 @@ profit. Hence, keep in mind that your performance is a mix of your strategies, your configuration, and the crypto-currency you have set up. ## Next step + Great, your strategy is profitable. What if the bot can give your the optimal parameters to use for your strategy? Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 713f23ecc..23bd7741a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,7 +6,8 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace -from typing import Dict, Tuple, Any, List, Optional +from datetime import datetime +from typing import Dict, Tuple, Any, List, Optional, NamedTuple import arrow from pandas import DataFrame @@ -23,6 +24,21 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) +class BacktestResult(NamedTuple): + """ + NamedTuple Defining BacktestResults inputs. + """ + pair: str + profit_percent: float + profit_abs: float + open_time: datetime + close_time: datetime + open_index: int + close_index: int + trade_duration: float + open_at_end: bool + + class Backtesting(object): """ Backtesting class, this class contains all the logic to run a backtest @@ -73,15 +89,15 @@ class Backtesting(object): headers = ['pair', 'buy count', 'avg profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: - result = results[results.currency == pair] + result = results[results.pair == pair] tabular_data.append([ pair, len(result.index), result.profit_percent.mean() * 100.0, - result.profit_BTC.sum(), - result.duration.mean(), - len(result[result.profit_BTC > 0]), - len(result[result.profit_BTC < 0]) + result.profit_abs.sum(), + result.trade_duration.mean(), + len(result[result.profit_abs > 0]), + len(result[result.profit_abs < 0]) ]) # Append Total @@ -89,16 +105,28 @@ class Backtesting(object): 'TOTAL', len(results.index), results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.duration.mean(), - len(results[results.profit_BTC > 0]), - len(results[results.profit_BTC < 0]) + results.profit_abs.sum(), + results.trade_duration.mean(), + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: + + records = [(trade_entry.pair, trade_entry.profit_percent, + trade_entry.open_time.timestamp(), + trade_entry.close_time.timestamp(), + trade_entry.open_index - 1, trade_entry.trade_duration) + for index, trade_entry in results.iterrows()] + + if records: + logger.info('Dumping backtest results to %s', recordfilename) + file_dump_json(recordfilename, records) + def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, - partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[Tuple]: + partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]: stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) @@ -121,15 +149,33 @@ class Backtesting(object): buy_signal = sell_row.buy if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal, sell_row.sell): - return \ - sell_row, \ - ( - pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close), - (sell_row.date - buy_row.date).seconds // 60 - ), \ - sell_row.date + + return BacktestResult(pair=pair, + profit_percent=trade.calc_profit_percent(rate=sell_row.close), + profit_abs=trade.calc_profit(rate=sell_row.close), + open_time=buy_row.date, + close_time=sell_row.date, + trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_index=buy_row.Index, + close_index=sell_row.Index, + open_at_end=False + ) + if partial_ticker: + # no sell condition found - trade stil open at end of backtest period + sell_row = partial_ticker[-1] + btr = BacktestResult(pair=pair, + profit_percent=trade.calc_profit_percent(rate=sell_row.close), + profit_abs=trade.calc_profit(rate=sell_row.close), + open_time=buy_row.date, + close_time=sell_row.date, + trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_index=buy_row.Index, + close_index=sell_row.Index, + open_at_end=True + ) + logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, + btr.profit_percent, btr.profit_abs) + return btr return None def backtest(self, args: Dict) -> DataFrame: @@ -145,17 +191,12 @@ class Backtesting(object): processed: a processed dictionary with format {pair, data} max_open_trades: maximum number of concurrent trades (default: 0, disabled) realistic: do we try to simulate realistic trades? (default: True) - sell_profit_only: sell if profit only - use_sell_signal: act on sell-signal :return: DataFrame """ headers = ['date', 'buy', 'open', 'close', 'sell'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) - record = args.get('record', None) - recordfilename = args.get('recordfn', 'backtest-result.json') - records = [] trades = [] trade_count_lock: Dict = {} for pair, pair_data in processed.items(): @@ -170,6 +211,8 @@ class Backtesting(object): ticker_data.drop(ticker_data.head(1).index, inplace=True) + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None @@ -187,28 +230,18 @@ class Backtesting(object): trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - ret = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - if ret: - row2, trade_entry, next_date = ret - lock_pair_until = next_date + if trade_entry: + lock_pair_until = trade_entry.close_time trades.append(trade_entry) - if record: - # Note, need to be json.dump friendly - # record a tuple of pair, current_profit_percent, - # entry-date, duration - records.append((pair, trade_entry[1], - row.date.strftime('%s'), - row2.date.strftime('%s'), - index, trade_entry[3])) - # For now export inside backtest(), maybe change so that backtest() - # returns a tuple like: (dataframe, records, logs, etc) - if record and record.find('trades') >= 0: - logger.info('Dumping backtest results to %s', recordfilename) - file_dump_json(recordfilename, records) - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] - return DataFrame.from_records(trades, columns=labels) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date + + return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: """ @@ -259,24 +292,22 @@ class Backtesting(object): ) # Execute backtest and print results - sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False) - use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False) results = self.backtest( { 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, 'realistic': self.config.get('realistic_simulation', False), - 'sell_profit_only': sell_profit_only, - 'use_sell_signal': use_sell_signal, - 'record': self.config.get('export'), - 'recordfn': self.config.get('exportfilename'), } ) + + if self.config.get('export', False): + self._store_backtest_result(self.config.get('exportfilename'), results) + logger.info( - '\n==================================== ' + '\n======================================== ' 'BACKTESTING REPORT' - ' ====================================\n' + ' =========================================\n' '%s', self._generate_text_table( data, @@ -284,6 +315,17 @@ class Backtesting(object): ) ) + logger.info( + '\n====================================== ' + 'LEFT OPEN TRADES REPORT' + ' ======================================\n' + '%s', + self._generate_text_table( + data, + results.loc[results.open_at_end] + ) + ) + def setup_configuration(args: Namespace) -> Dict[str, Any]: """ diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 978d1fe2c..19e732d7b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -449,7 +449,7 @@ class Hyperopt(Backtesting): total_profit = results.profit_percent.sum() trade_count = len(results.index) - trade_duration = results.duration.mean() + trade_duration = results.trade_duration.mean() if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') @@ -486,10 +486,10 @@ class Hyperopt(Backtesting): 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), + results.profit_abs.sum(), self.config['stake_currency'], results.profit_percent.sum(), - results.duration.mean(), + results.trade_duration.mean(), ) def start(self) -> None: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e1876d15b..31080c503 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -353,10 +353,10 @@ def test_generate_text_table(default_conf, mocker): results = pd.DataFrame( { - 'currency': ['ETH/BTC', 'ETH/BTC'], + 'pair': ['ETH/BTC', 'ETH/BTC'], 'profit_percent': [0.1, 0.2], - 'profit_BTC': [0.2, 0.4], - 'duration': [10, 30], + 'profit_abs': [0.2, 0.4], + 'trade_duration': [10, 30], 'profit': [2, 0], 'loss': [0, 0] } @@ -469,6 +469,7 @@ def test_backtest(default_conf, fee, mocker) -> None: } ) assert not results.empty + assert len(results) == 2 def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: @@ -491,6 +492,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: } ) assert not results.empty + assert len(results) == 1 def test_processed(default_conf, mocker) -> None: @@ -512,7 +514,7 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) - tests = [['raise', 17], ['lower', 0], ['sine', 16]] + tests = [['raise', 18], ['lower', 0], ['sine', 16]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres, mocker) @@ -572,7 +574,10 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): backtesting.populate_buy_trend = _trend_alternate # Override backtesting.populate_sell_trend = _trend_alternate # Override results = backtesting.backtest(backtest_conf) - assert len(results) == 3 + backtesting._store_backtest_result("test_.json", results) + assert len(results) == 4 + # One trade was force-closed at the end + assert len(results.loc[results.open_at_end]) == 1 def test_backtest_record(default_conf, fee, mocker): @@ -584,22 +589,30 @@ def test_backtest_record(default_conf, fee, mocker): 'freqtrade.optimize.backtesting.file_dump_json', new=lambda n, r: (names.append(n), records.append(r)) ) - backtest_conf = _make_backtest_conf( - mocker, - conf=default_conf, - pair='UNITTEST/BTC', - record="trades" - ) + backtesting = Backtesting(default_conf) - backtesting.populate_buy_trend = _trend_alternate # Override - backtesting.populate_sell_trend = _trend_alternate # Override - results = backtesting.backtest(backtest_conf) - assert len(results) == 3 + results = pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_index": [1, 119, 153, 185], + "close_index": [118, 151, 184, 199], + "trade_duration": [123, 34, 31, 14]}) + backtesting._store_backtest_result("backtest-result.json", results) + assert len(results) == 4 # Assert file_dump_json was only called once assert names == ['backtest-result.json'] records = records[0] # Ensure records are of correct type - assert len(records) == 3 + assert len(records) == 4 # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index ce7cd77c8..f8dce5fd6 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -362,7 +362,7 @@ def test_format_results(init_hyperopt): ('LTC/BTC', 1, 1, 123), ('XPR/BTC', -1, -2, -246) ] - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration'] df = pd.DataFrame.from_records(trades, columns=labels) result = _HYPEROPT.format_results(df) @@ -492,7 +492,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: trades = [ ('POWR/BTC', 0.023117, 0.000233, 100) ] - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration'] backtest_result = pd.DataFrame.from_records(trades, columns=labels) mocker.patch(