Merge pull request #2344 from freqtrade/backtest_nofees

Backtest no fees / custom fees

docs/backtesting.md

@@ -72,6 +72,17 @@ The exported trades can be used for [further analysis](#further-backtest-result-
 freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json
 ```
 
+#### Supplying custom fee value
+
+Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt.
+To account for this in backtesting, you can use `--fee 0.001` to supply this value to backtesting.
+This fee must be a percentage, and will be applied twice (once for trade entry, and once for trade exit).
+
+```bash
+freqtrade backtesting --fee 0.001
+```
+
+
 #### Running backtest with smaller testset by using timerange
 
 Use the `--timerange` argument to change how much of the testset you want to use.

docs/bot-usage.md

@@ -174,22 +174,25 @@ Backtesting also uses the config specified via `-c/--config`.
 
 ```
 usage: freqtrade backtesting [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                           [--max_open_trades MAX_OPEN_TRADES]
+                             [--max_open_trades INT]
-                           [--stake_amount STAKE_AMOUNT] [-r] [--eps] [--dmmp]
+                             [--stake_amount STAKE_AMOUNT] [--fee FLOAT]
-                           [-l]
+                             [--eps] [--dmmp]
                              [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
                              [--export EXPORT] [--export-filename PATH]
 
 optional arguments:
   -h, --help            show this help message and exit
   -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL
-                        Specify ticker interval (1m, 5m, 30m, 1h, 1d).
+                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
+                        `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
-  --max_open_trades MAX_OPEN_TRADES
+  --max_open_trades INT
                         Specify max_open_trades to use.
   --stake_amount STAKE_AMOUNT
                         Specify stake_amount.
+  --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
+                        entry and exit).
   --eps, --enable-position-stacking
                         Allow buying the same pair multiple times (position
                         stacking).
@@ -199,19 +202,21 @@ optional arguments:
                         number).
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a space-separated list of strategies to
-                        backtest Please note that ticker-interval needs to be
+                        backtest. Please note that ticker-interval needs to be
                         set either in config or via command line. When using
-                        this together with --export trades, the strategy-name
+                        this together with `--export trades`, the strategy-
-                        is injected into the filename (so backtest-data.json
+                        name is injected into the filename (so `backtest-
-                        becomes backtest-data-DefaultStrategy.json
+                        data.json` becomes `backtest-data-
-  --export EXPORT       Export backtest results, argument are: trades. Example
+                        DefaultStrategy.json`
-                        --export=trades
+  --export EXPORT       Export backtest results, argument are: trades.
+                        Example: `--export=trades`
   --export-filename PATH
-                        Save backtest results to this filename requires
+                        Save backtest results to the file with this filename
-                        --export to be set as well Example --export-
+                        (default: `user_data/backtest_results/backtest-
-                        filename=user_data/backtest_results/backtest_today.json
+                        result.json`). Requires `--export` to be set as well.
-                        (default: user_data/backtest_results/backtest-
+                        Example: `--export-filename=user_data/backtest_results
-                        result.json)
+                        /backtest_today.json`
+
 ```
 
 ### Getting historic data for backtesting
@@ -228,13 +233,13 @@ to find optimal parameter values for your stategy.
 ```
 usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
                           [--max_open_trades INT]
-                          [--stake_amount STAKE_AMOUNT] [-r]
+                          [--stake_amount STAKE_AMOUNT] [--fee FLOAT]
                           [--customhyperopt NAME] [--hyperopt-path PATH]
                           [--eps] [-e INT]
                           [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]]
-                          [--dmmp] [--print-all] [--no-color] [-j JOBS]
+                          [--dmmp] [--print-all] [--no-color] [--print-json]
-                          [--random-state INT] [--min-trades INT] [--continue]
+                          [-j JOBS] [--random-state INT] [--min-trades INT]
-                          [--hyperopt-loss NAME]
+                          [--continue] [--hyperopt-loss NAME]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -247,6 +252,8 @@ optional arguments:
                         Specify max_open_trades to use.
   --stake_amount STAKE_AMOUNT
                         Specify stake_amount.
+  --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
+                        entry and exit).
   --customhyperopt NAME
                         Specify hyperopt class name (default:
                         `DefaultHyperOpts`).
@@ -266,6 +273,7 @@ optional arguments:
   --print-all           Print all results, not only the best ones.
   --no-color            Disable colorization of hyperopt results. May be
                         useful if you are redirecting output to a file.
+  --print-json          Print best result detailization in JSON format.
   -j JOBS, --job-workers JOBS
                         The number of concurrently running jobs for
                         hyperoptimization (hyperopt worker processes). If -1
@@ -284,8 +292,8 @@ optional arguments:
                         generate completely different results, since the
                         target for optimization is different. Built-in
                         Hyperopt-loss-functions are: DefaultHyperOptLoss,
-                        OnlyProfitHyperOptLoss, SharpeHyperOptLoss.
+                        OnlyProfitHyperOptLoss, SharpeHyperOptLoss.(default:
-                        (default: `DefaultHyperOptLoss`).
+                        `DefaultHyperOptLoss`).
 ```
 
 ## Edge commands
@@ -294,25 +302,28 @@ To know your trade expectancy and winrate against historical data, you can use E
 
 ```
 usage: freqtrade edge [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                    [--max_open_trades MAX_OPEN_TRADES]
+                      [--max_open_trades INT] [--stake_amount STAKE_AMOUNT]
-                    [--stake_amount STAKE_AMOUNT] [-r]
+                      [--fee FLOAT] [--stoplosses STOPLOSS_RANGE]
-                    [--stoplosses STOPLOSS_RANGE]
 
 optional arguments:
   -h, --help            show this help message and exit
   -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL
-                        Specify ticker interval (1m, 5m, 30m, 1h, 1d).
+                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
+                        `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
-  --max_open_trades MAX_OPEN_TRADES
+  --max_open_trades INT
                         Specify max_open_trades to use.
   --stake_amount STAKE_AMOUNT
                         Specify stake_amount.
+  --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
+                        entry and exit).
   --stoplosses STOPLOSS_RANGE
-                        Defines a range of stoploss against which edge will
+                        Defines a range of stoploss values against which edge
-                        assess the strategy the format is "min,max,step"
+                        will assess the strategy. The format is "min,max,step"
-                        (without any space).example:
+                        (without any space). Example:
-                        --stoplosses=-0.01,-0.1,-0.001
+                        `--stoplosses=-0.01,-0.1,-0.001`
+
 ```
 
 To understand edge and how to read the results, please read the [edge documentation](edge.md).

freqtrade/configuration/arguments.py

@@ -15,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
 ARGS_MAIN = ARGS_COMMON + ARGS_STRATEGY + ["db_url", "sd_notify"]
 
 ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange",
-                        "max_open_trades", "stake_amount"]
+                        "max_open_trades", "stake_amount", "fee"]
 
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
                                         "strategy_list", "export", "exportfilename"]

freqtrade/configuration/cli_options.py

@@ -144,6 +144,12 @@ AVAILABLE_CLI_OPTIONS = {
         default=os.path.join('user_data', 'backtest_results',
                              'backtest-result.json'),
     ),
+    "fee": Arg(
+        '--fee',
+        help='Specify fee ratio. Will be applied twice (on trade entry and exit).',
+        type=float,
+        metavar='FLOAT',
+    ),
     # Edge
     "stoploss_range": Arg(
         '--stoplosses',

freqtrade/configuration/configuration.py

@@ -210,6 +210,10 @@ class Configuration:
                              logstring='Parameter --stake_amount detected, '
                              'overriding stake_amount to: {} ...')
 
+        self._args_to_config(config, argname='fee',
+                             logstring='Parameter --fee detected, '
+                             'setting fee to: {} ...')
+
         self._args_to_config(config, argname='timerange',
                              logstring='Parameter --timerange detected: {} ...')
 
@@ -323,7 +327,8 @@ class Configuration:
                         sample: logfun=len (prints the length of the found
                         configuration instead of the content)
         """
-        if argname in self.args and self.args[argname]:
+        if (argname in self.args and self.args[argname] is not None
+           and self.args[argname] is not False):
 
             config.update({argname: self.args[argname]})
             if logfun:

freqtrade/edge/__init__.py

@@ -77,7 +77,9 @@ class Edge:
 
         self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(
             days=-1 * self._since_number_of_days).format('YYYYMMDD'))
-
+        if config.get('fee'):
+            self.fee = config['fee']
+        else:
             self.fee = self.exchange.get_fee()
 
     def calculate(self) -> bool:

freqtrade/optimize/backtesting.py

@@ -63,8 +63,11 @@ class Backtesting:
         self.config['exchange']['uid'] = ''
         self.config['dry_run'] = True
         self.strategylist: List[IStrategy] = []
-
         self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
+
+        if config.get('fee'):
+            self.fee = config['fee']
+        else:
             self.fee = self.exchange.get_fee()
 
         if self.config.get('runmode') != RunMode.HYPEROPT:

tests/optimize/test_backtesting.py

@@ -26,6 +26,21 @@ from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
 
+ORDER_TYPES = [
+    {
+        'buy': 'limit',
+        'sell': 'limit',
+        'stoploss': 'limit',
+        'stoploss_on_exchange': False
+    },
+    {
+        'buy': 'limit',
+        'sell': 'limit',
+        'stoploss': 'limit',
+        'stoploss_on_exchange': True
+    }]
+
+
 def trim_dictlist(dict_list, num):
     new = {}
     for pair, pair_data in dict_list.items():
@@ -211,7 +226,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
         '--disable-max-market-positions',
         '--timerange', ':100',
         '--export', '/bar/foo',
-        '--export-filename', 'foo_bar.json'
+        '--export-filename', 'foo_bar.json',
+        '--fee', '0',
     ]
 
     config = setup_configuration(get_args(args), RunMode.BACKTEST)
@@ -243,6 +259,9 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
     assert 'exportfilename' in config
     assert log_has('Storing backtest results to {} ...'.format(config['exportfilename']), caplog)
 
+    assert 'fee' in config
+    assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog)
+
 
 def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None:
     default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
@@ -277,21 +296,6 @@ def test_start(mocker, fee, default_conf, caplog) -> None:
     assert start_mock.call_count == 1
 
 
-ORDER_TYPES = [
-    {
-        'buy': 'limit',
-        'sell': 'limit',
-        'stoploss': 'limit',
-        'stoploss_on_exchange': False
-    },
-    {
-        'buy': 'limit',
-        'sell': 'limit',
-        'stoploss': 'limit',
-        'stoploss_on_exchange': True
-    }]
-
-
 @pytest.mark.parametrize("order_types", ORDER_TYPES)
 def test_backtesting_init(mocker, default_conf, order_types) -> None:
     """
@@ -314,10 +318,6 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None:
 
 
 def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> None:
-    """
-    Check that stoploss_on_exchange is set to False while backtesting
-    since backtesting assumes a perfect stoploss anyway.
-    """
     patch_exchange(mocker)
     del default_conf['ticker_interval']
     default_conf['strategy_list'] = ['DefaultStrategy',
@@ -330,6 +330,16 @@ def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> No
             "or as cli argument `--ticker-interval 5m`", caplog)
 
 
+def test_tickerdata_with_fee(default_conf, mocker, testdatadir) -> None:
+    patch_exchange(mocker)
+    default_conf['fee'] = 0.1234
+
+    fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5))
+    backtesting = Backtesting(default_conf)
+    assert backtesting.fee == 0.1234
+    assert fee_mock.call_count == 0
+
+
 def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
     patch_exchange(mocker)
     timerange = TimeRange(None, 'line', 0, -100)

tests/optimize/test_edge_cli.py

@@ -98,6 +98,16 @@ def test_edge_init(mocker, edge_conf) -> None:
     assert callable(edge_cli.edge.calculate)
 
 
+def test_edge_init_fee(mocker, edge_conf) -> None:
+    patch_exchange(mocker)
+    edge_conf['fee'] = 0.1234
+    edge_conf['stake_amount'] = 20
+    fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5))
+    edge_cli = EdgeCli(edge_conf)
+    assert edge_cli.edge.fee == 0.1234
+    assert fee_mock.call_count == 0
+
+
 def test_generate_edge_table(edge_conf, mocker):
     patch_exchange(mocker)
     edge_cli = EdgeCli(edge_conf)