Fix the fee calculation, backtesting, and hyperopt fee calculation and avg_profit

.travis.yml

@@ -1,4 +1,4 @@
-sudo: false
+sudo: true
 os:
 - linux
 language: python
@@ -11,9 +11,7 @@ addons:
     - libdw-dev
     - binutils-dev
 install:
-- wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
+- ./install_ta-lib.sh
-- tar zxvf ta-lib-0.4.0-src.tar.gz
-- cd ta-lib && ./configure && sudo make && sudo make install && cd ..
 - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 - pip install flake8 coveralls
 - pip install -r requirements.txt
@@ -27,11 +25,13 @@ jobs:
     - script:
       - cp config.json.example config.json
       - python freqtrade/main.py hyperopt -e 5
+    - script: flake8 freqtrade
 after_success:
-- flake8 freqtrade && coveralls
+- coveralls
 notifications:
   slack:
     secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q=
 cache:
   directories:
     - $HOME/.cache/pip
+    - ta-lib

freqtrade/exchange/bittrex.py

@@ -50,8 +50,8 @@ class Bittrex(Exchange):
 
     @property
     def fee(self) -> float:
-        # See https://bittrex.com/fees
+        # 0.25 %: See https://bittrex.com/fees
-        return 0.0025  #0.25%
+        return 0.0025
 
     def buy(self, pair: str, rate: float, amount: float) -> str:
         data = _API.buy_limit(pair.replace('_', '-'), amount, rate)

freqtrade/misc.py

@@ -168,8 +168,8 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
     )
     backtesting_cmd.add_argument(
         '-r', '--refresh-pairs-cached',
-        help='refresh the pairs files in tests/testdata with the latest data from Bittrex. Use it if you want to \
+        help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
-              run your backtesting with up-to-date data.',
+              Use it if you want to run your backtesting with up-to-date data.',
         action='store_true',
         dest='refresh_pairs',
     )

freqtrade/optimize/__init__.py

@@ -13,7 +13,8 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
 logger = logging.getLogger(__name__)
 
 
-def load_data(pairs: List[str], ticker_interval: int = 5, refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
+def load_data(pairs: List[str], ticker_interval: int = 5,
+              refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
     """
     Loads ticker history data for the given parameters
     :param ticker_interval: ticker interval in minutes
@@ -61,10 +62,10 @@ def download_pairs(pairs: List[str]) -> bool:
     """For each pairs passed in parameters, download 1 and 5 ticker intervals"""
     for pair in pairs:
         try:
-            for interval in [1,5]:
+            for interval in [1, 5]:
                 download_backtesting_testdata(pair=pair, interval=interval)
         except BaseException:
-            logger.info('Impossible to download the pair: "{pair}", Interval: {interval} min'.format(
+            logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format(
                 pair=pair,
                 interval=interval,
             ))
@@ -103,7 +104,7 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool:
         logger.debug("Current Start: None")
         logger.debug("Current End: None")
 
-    new_data = get_ticker_history(pair = pair, tick_interval = int(interval))
+    new_data = get_ticker_history(pair=pair, tick_interval=int(interval))
     for row in new_data:
         if row not in data:
             data.append(row)

freqtrade/optimize/backtesting.py

@@ -48,8 +48,8 @@ def generate_text_table(
         tabular_data.append([
             pair,
             len(result.index),
-            '{:.2f}%'.format(result.profit.mean() * 100.0),
+            '{:.2f}%'.format(result.profit_percent.mean() * 100.0),
-            '{:.08f} {}'.format(result.profit.sum(), stake_currency),
+            '{:.08f} {}'.format(result.profit_BTC.sum(), stake_currency),
             '{:.2f}'.format(result.duration.mean() * ticker_interval),
         ])
 
@@ -57,8 +57,8 @@ def generate_text_table(
     tabular_data.append([
         'TOTAL',
         len(results.index),
-        '{:.2f}%'.format(results.profit.mean() * 100.0),
+        '{:.2f}%'.format(results.profit_percent.mean() * 100.0),
-        '{:.08f} {}'.format(results.profit.sum(), stake_currency),
+        '{:.08f} {}'.format(results.profit_BTC.sum(), stake_currency),
         '{:.2f}'.format(results.duration.mean() * ticker_interval),
     ])
     return tabulate(tabular_data, headers=headers)
@@ -98,7 +98,8 @@ def backtest(config: Dict, processed: Dict[str, DataFrame],
             trade = Trade(
                 open_rate=row.close,
                 open_date=row.date,
-                amount=config['stake_amount'],
+                stake_amount=config['stake_amount'],
+                amount=config['stake_amount'] / row.open,
                 fee=exchange.get_fee()
             )
 
@@ -109,12 +110,20 @@ def backtest(config: Dict, processed: Dict[str, DataFrame],
                     trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1
 
                 if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1:
-                    current_profit = trade.calc_profit_percent(row2.close)
+                    current_profit_percent = trade.calc_profit_percent(rate=row2.close)
+                    current_profit_BTC = trade.calc_profit(rate=row2.close)
                     lock_pair_until = row2.Index
 
-                    trades.append((pair, current_profit, row2.Index - row.Index))
+                    trades.append(
+                        (
+                            pair,
+                            current_profit_percent,
+                            current_profit_BTC,
+                            row2.Index - row.Index
+                         )
+                    )
                     break
-    labels = ['currency', 'profit', 'duration']
+    labels = ['currency', 'profit_percent', 'profit_BTC', 'duration']
     return DataFrame.from_records(trades, columns=labels)
 
 
@@ -140,7 +149,8 @@ def start(args):
             data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
         logger.info('Using local backtesting data (using whitelist in given config) ...')
-        data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs)
+        data = load_data(pairs=pairs, ticker_interval=args.ticker_interval,
+                         refresh_pairs=args.refresh_pairs)
 
         logger.info('Using stake_currency: %s ...', config['stake_currency'])
         logger.info('Using stake_amount: %s ...', config['stake_amount'])

freqtrade/optimize/hyperopt.py

@@ -131,7 +131,7 @@ def optimizer(params):
 
     result = format_results(results)
 
-    total_profit = results.profit.sum() * 1000
+    total_profit = results.profit_percent.sum() * 1000
     trade_count = len(results.index)
 
     trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
@@ -144,13 +144,13 @@ def optimizer(params):
         'total_profit': total_profit,
         'trade_loss': trade_loss,
         'profit_loss': profit_loss,
-        'avg_profit': results.profit.mean() * 100.0,
+        'avg_profit': results.profit_percent.mean() * 100.0,
         'avg_duration': results.duration.mean() * 5,
         'current_tries': _CURRENT_TRIES,
         'total_tries': TOTAL_TRIES,
         'result': result,
         'results': results
-        }
+    }
 
     # logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result))
     log_results(result_data)
@@ -166,10 +166,10 @@ def format_results(results: DataFrame):
     return ('Made {:6d} buys. Average profit {: 5.2f}%. '
             'Total profit was {: 7.3f}. Average duration {:5.1f} mins.').format(
                 len(results.index),
-                results.profit.mean() * 100.0,
+                results.profit_percent.mean() * 100.0,
-                results.profit.sum(),
+                results.profit_BTC.sum(),
                 results.duration.mean() * 5,
-            )
+    )
 
 
 def buy_strategy_generator(params):
@@ -232,7 +232,8 @@ def start(args):
     logger.info('Using config: %s ...', args.config)
     config = load_config(args.config)
     pairs = config['exchange']['pair_whitelist']
-    PROCESSED = optimize.preprocess(optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval))
+    PROCESSED = optimize.preprocess(optimize.load_data(
+        pairs=pairs, ticker_interval=args.ticker_interval))
 
     if args.mongodb:
         logger.info('Using mongodb ...')

freqtrade/persistence.py

@@ -121,7 +121,9 @@ class Trade(_DECL_BASE):
             self
         )
 
-    def calc_open_trade_price(self, fee: Optional[float] = None) -> float:
+    def calc_open_trade_price(
+            self,
+            fee: Optional[float] = None) -> float:
         """
         Calculate the open_rate in BTC
         :param fee: fee to use on the open rate (optional).
@@ -134,7 +136,10 @@ class Trade(_DECL_BASE):
         fees = buy_trade * Decimal(fee or self.fee)
         return float(buy_trade + fees)
 
-    def calc_close_trade_price(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float:
+    def calc_close_trade_price(
+            self,
+            rate: Optional[float] = None,
+            fee: Optional[float] = None) -> float:
         """
         Calculate the close_rate in BTC
         :param fee: fee to use on the close rate (optional).
@@ -152,7 +157,10 @@ class Trade(_DECL_BASE):
         fees = sell_trade * Decimal(fee or self.fee)
         return float(sell_trade - fees)
 
-    def calc_profit(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float:
+    def calc_profit(
+            self,
+            rate: Optional[float] = None,
+            fee: Optional[float] = None) -> float:
         """
         Calculate the profit in BTC between Close and Open trade
         :param fee: fee to use on the close rate (optional).
@@ -168,7 +176,10 @@ class Trade(_DECL_BASE):
         )
         return float("{0:.8f}".format(close_trade_price - open_trade_price))
 
-    def calc_profit_percent(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float:
+    def calc_profit_percent(
+            self,
+            rate: Optional[float] = None,
+            fee: Optional[float] = None) -> float:
         """
         Calculates the profit in percentage (including fee).
         :param rate: rate to compare with (optional).

freqtrade/rpc/telegram.py

@@ -232,8 +232,8 @@ def _daily(bot: Bot, update: Update) -> None:
 
     for day in range(0, timescale):
         # need to query between day+1 and day-1
-        nextdate = date.fromordinal(today-day+1)
+        nextdate = date.fromordinal(today - day + 1)
-        prevdate = date.fromordinal(today-day-1)
+        prevdate = date.fromordinal(today - day - 1)
         trades = Trade.query \
             .filter(Trade.is_open.is_(False)) \
             .filter(between(Trade.close_date, prevdate, nextdate)) \

freqtrade/tests/conftest.py

@@ -66,6 +66,7 @@ def ticker():
         'last': 0.00001098,
     })
 
+
 @pytest.fixture
 def ticker_sell_up():
     return MagicMock(return_value={
@@ -74,6 +75,7 @@ def ticker_sell_up():
         'last': 0.00001172,
     })
 
+
 @pytest.fixture
 def ticker_sell_down():
     return MagicMock(return_value={
@@ -82,6 +84,7 @@ def ticker_sell_down():
         'last': 0.00001044,
     })
 
+
 @pytest.fixture
 def health():
     return MagicMock(return_value=[{
@@ -143,7 +146,7 @@ def limit_sell_order():
 @pytest.fixture
 def ticker_history():
     return [
-	    {
+        {
             "O": 8.794e-05,
             "H": 8.948e-05,
             "L": 8.794e-05,
@@ -152,7 +155,7 @@ def ticker_history():
             "T": "2017-11-26T08:50:00",
             "BV": 0.0877869
         },
-	    {
+        {
             "O": 8.88e-05,
             "H": 8.942e-05,
             "L": 8.88e-05,
@@ -161,7 +164,7 @@ def ticker_history():
             "T": "2017-11-26T08:55:00",
             "BV": 0.05874751
         },
-	    {
+        {
             "O": 8.891e-05,
             "H": 8.893e-05,
             "L": 8.875e-05,

freqtrade/tests/test_optimize_backtesting.py

@@ -1,15 +1,11 @@
 # pragma pylint: disable=missing-docstring,W0212
 
-from unittest.mock import MagicMock
-
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
 from freqtrade.optimize.backtesting import backtest
 from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata
 import os
 
-import pytest
-
 
 def test_backtest(default_conf, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
@@ -30,6 +26,7 @@ def test_1min_ticker_interval(default_conf, mocker):
     results = backtest(default_conf, optimize.preprocess(data), 1, True)
     assert len(results) > 0
 
+
 def test_backtest_with_new_pair(default_conf, ticker_history, mocker):
     mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history)
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
@@ -59,7 +56,7 @@ def test_download_pairs(default_conf, ticker_history, mocker):
     file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json'
     file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json'
 
-    assert download_pairs(pairs = ['BTC-MEME', 'BTC-CFI']) is True
+    assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True
 
     assert os.path.isfile(file1_1) is True
     assert os.path.isfile(file1_5) is True
@@ -87,7 +84,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
 
     # Download a 1 min ticker file
     file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json'
-    download_backtesting_testdata(pair = "BTC-XEL", interval = 1)
+    download_backtesting_testdata(pair="BTC-XEL", interval=1)
     assert os.path.isfile(file1) is True
 
     if os.path.isfile(file1):
@@ -95,7 +92,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
 
     # Download a 5 min ticker file
     file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json'
-    download_backtesting_testdata(pair = "BTC-STORJ", interval = 5)
+    download_backtesting_testdata(pair="BTC-STORJ", interval=5)
     assert os.path.isfile(file2) is True
 
     if os.path.isfile(file2):

freqtrade/tests/test_persistence.py

@@ -10,17 +10,23 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order):
     On this test we will buy and sell a crypto currency.
 
     Buy
-    - Buy: 90.99181073 Crypto at 0.00001099 BTC (90.99181073*0.00001099 = 0.0009999 BTC)
+    - Buy: 90.99181073 Crypto at 0.00001099 BTC
+        (90.99181073*0.00001099 = 0.0009999 BTC)
     - Buying fee: 0.25%
-    - Total cost of buy trade: 0.001002500 BTC ((90.99181073*0.00001099) + ((90.99181073*0.00001099)*0.0025))
+    - Total cost of buy trade: 0.001002500 BTC
+        ((90.99181073*0.00001099) + ((90.99181073*0.00001099)*0.0025))
 
     Sell
-    - Sell: 90.99181073 Crypto at 0.00001173 BTC (90.99181073*0.00001173 = 0,00106733394 BTC)
+    - Sell: 90.99181073 Crypto at 0.00001173 BTC
+        (90.99181073*0.00001173 = 0,00106733394 BTC)
     - Selling fee: 0.25%
-    - Total cost of sell trade: 0.001064666 BTC ((90.99181073*0.00001173) - ((90.99181073*0.00001173)*0.0025))
+    - Total cost of sell trade: 0.001064666 BTC
+        ((90.99181073*0.00001173) - ((90.99181073*0.00001173)*0.0025))
 
-    Profit/Loss: +0.000062166 BTC (Sell:0.001064666 - Buy:0.001002500)
+    Profit/Loss: +0.000062166 BTC
-    Profit/Loss percentage: 0.0620  ((0.001064666/0.001002500)-1 = 6.20%)
+        (Sell:0.001064666 - Buy:0.001002500)
+    Profit/Loss percentage: 0.0620
+        ((0.001064666/0.001002500)-1 = 6.20%)
 
     :param limit_buy_order:
     :param limit_sell_order:

freqtrade/tests/test_rpc_telegram.py

@@ -1,6 +1,6 @@
 # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103
 import re
-from datetime import datetime, date
+from datetime import datetime
 from random import randint
 from unittest.mock import MagicMock
 
@@ -151,7 +151,8 @@ def test_status_table_handle(default_conf, update, ticker, mocker):
     assert msg_mock.call_count == 1
 
 
-def test_profit_handle(default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker):
+def test_profit_handle(
+        default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True)
     msg_mock = MagicMock()
@@ -246,7 +247,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m
     # Create some test data
     create_trade(0.001)
 
-    ## Decrease the price and sell it
+    # Decrease the price and sell it
     mocker.patch.multiple('freqtrade.main.exchange',
                           validate_pairs=MagicMock(),
                           get_ticker=ticker_sell_down)
@@ -383,7 +384,6 @@ def test_performance_handle(
     assert '<code>BTC_ETH\t6.20%</code>' in msg_mock.call_args_list[0][0][0]
 
 
-
 def test_daily_handle(
         default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)

install_ta-lib.sh

@@ -0,0 +1,8 @@
+if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then
+  curl -O http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
+  tar zxvf ta-lib-0.4.0-src.tar.gz
+  cd ta-lib && ./configure && make && sudo make install && cd ..
+else
+  echo "TA-lib already installed, skipping download and build."
+  cd ta-lib && sudo make install && cd ..
+fi

scripts/plot_dataframe.py

@@ -3,7 +3,6 @@
 import matplotlib  # Install PYQT5 manually if you want to test this helper function
 matplotlib.use("Qt5Agg")
 import matplotlib.pyplot as plt
-
 from freqtrade import exchange, analyze
 
 
@@ -52,4 +51,3 @@ def plot_analyzed_dataframe(pair: str) -> None:
 
 if __name__ == '__main__':
     plot_analyzed_dataframe('BTC_ETH')
-