adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix the fee calculation, backtesting, and hyperopt fee calculation and avg_profit

Browse Source
This commit is contained in:
Gerald Lonlas 2017-12-18 21:58:02 -08:00
commit d258118b0a
14 changed files with 94 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.travis.yml
View File

@ -1,4 +1,4 @@
sudo: false
sudo: true
os:
- linux
language: python
@ -11,9 +11,7 @@ addons:
- libdw-dev
- binutils-dev
install:
- wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
- tar zxvf ta-lib-0.4.0-src.tar.gz
- cd ta-lib && ./configure && sudo make && sudo make install && cd ..
- ./install_ta-lib.sh
- export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
- pip install flake8 coveralls
- pip install -r requirements.txt
@ -27,11 +25,13 @@ jobs:
- script:
- cp config.json.example config.json
- python freqtrade/main.py hyperopt -e 5
- script: flake8 freqtrade
after_success:
- flake8 freqtrade && coveralls
- coveralls
notifications:
slack:
secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q=
cache:
directories:
- $HOME/.cache/pip
- $HOME/.cache/pip
- ta-lib

4
freqtrade/exchange/bittrex.py
View File

@ -50,8 +50,8 @@ class Bittrex(Exchange):
@property
def fee(self) -> float:
# See https://bittrex.com/fees
return 0.0025 #0.25%
# 0.25 %: See https://bittrex.com/fees
return 0.0025
def buy(self, pair: str, rate: float, amount: float) -> str:
data = _API.buy_limit(pair.replace('_', '-'), amount, rate)

4
freqtrade/misc.py
View File

@ -168,8 +168,8 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
)
backtesting_cmd.add_argument(
'-r', '--refresh-pairs-cached',
help='refresh the pairs files in tests/testdata with the latest data from Bittrex. Use it if you want to \
run your backtesting with up-to-date data.',
help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
Use it if you want to run your backtesting with up-to-date data.',
action='store_true',
dest='refresh_pairs',
)

9
freqtrade/optimize/__init__.py
View File

@ -13,7 +13,8 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
logger = logging.getLogger(__name__)
def load_data(pairs: List[str], ticker_interval: int = 5, refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
def load_data(pairs: List[str], ticker_interval: int = 5,
refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
"""
Loads ticker history data for the given parameters
:param ticker_interval: ticker interval in minutes
@ -61,10 +62,10 @@ def download_pairs(pairs: List[str]) -> bool:
"""For each pairs passed in parameters, download 1 and 5 ticker intervals"""
for pair in pairs:
try:
for interval in [1,5]:
for interval in [1, 5]:
download_backtesting_testdata(pair=pair, interval=interval)
except BaseException:
logger.info('Impossible to download the pair: "{pair}", Interval: {interval} min'.format(
logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format(
pair=pair,
interval=interval,
))
@ -103,7 +104,7 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool:
logger.debug("Current Start: None")
logger.debug("Current End: None")
new_data = get_ticker_history(pair = pair, tick_interval = int(interval))
new_data = get_ticker_history(pair=pair, tick_interval=int(interval))
for row in new_data:
if row not in data:
data.append(row)

28
freqtrade/optimize/backtesting.py
View File

@ -48,8 +48,8 @@ def generate_text_table(
tabular_data.append([
pair,
len(result.index),
'{:.2f}%'.format(result.profit.mean() * 100.0),
'{:.08f} {}'.format(result.profit.sum(), stake_currency),
'{:.2f}%'.format(result.profit_percent.mean() * 100.0),
'{:.08f} {}'.format(result.profit_BTC.sum(), stake_currency),
'{:.2f}'.format(result.duration.mean() * ticker_interval),
])
@ -57,8 +57,8 @@ def generate_text_table(
tabular_data.append([
'TOTAL',
len(results.index),
'{:.2f}%'.format(results.profit.mean() * 100.0),
'{:.08f} {}'.format(results.profit.sum(), stake_currency),
'{:.2f}%'.format(results.profit_percent.mean() * 100.0),
'{:.08f} {}'.format(results.profit_BTC.sum(), stake_currency),
'{:.2f}'.format(results.duration.mean() * ticker_interval),
])
return tabulate(tabular_data, headers=headers)
@ -98,7 +98,8 @@ def backtest(config: Dict, processed: Dict[str, DataFrame],
trade = Trade(
open_rate=row.close,
open_date=row.date,
amount=config['stake_amount'],
stake_amount=config['stake_amount'],
amount=config['stake_amount'] / row.open,
fee=exchange.get_fee()
)
@ -109,12 +110,20 @@ def backtest(config: Dict, processed: Dict[str, DataFrame],
trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1
if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1:
current_profit = trade.calc_profit_percent(row2.close)
current_profit_percent = trade.calc_profit_percent(rate=row2.close)
current_profit_BTC = trade.calc_profit(rate=row2.close)
lock_pair_until = row2.Index
trades.append((pair, current_profit, row2.Index - row.Index))
trades.append(
(
pair,
current_profit_percent,
current_profit_BTC,
row2.Index - row.Index
)
)
break
labels = ['currency', 'profit', 'duration']
labels = ['currency', 'profit_percent', 'profit_BTC', 'duration']
return DataFrame.from_records(trades, columns=labels)
@ -140,7 +149,8 @@ def start(args):
data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
else:
logger.info('Using local backtesting data (using whitelist in given config) ...')
data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs)
data = load_data(pairs=pairs, ticker_interval=args.ticker_interval,
refresh_pairs=args.refresh_pairs)
logger.info('Using stake_currency: %s ...', config['stake_currency'])
logger.info('Using stake_amount: %s ...', config['stake_amount'])

15
freqtrade/optimize/hyperopt.py
View File

@ -131,7 +131,7 @@ def optimizer(params):
result = format_results(results)
total_profit = results.profit.sum() * 1000
total_profit = results.profit_percent.sum() * 1000
trade_count = len(results.index)
trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
@ -144,13 +144,13 @@ def optimizer(params):
'total_profit': total_profit,
'trade_loss': trade_loss,
'profit_loss': profit_loss,
'avg_profit': results.profit.mean() * 100.0,
'avg_profit': results.profit_percent.mean() * 100.0,
'avg_duration': results.duration.mean() * 5,
'current_tries': _CURRENT_TRIES,
'total_tries': TOTAL_TRIES,
'result': result,
'results': results
}
}
# logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result))
log_results(result_data)
@ -166,10 +166,10 @@ def format_results(results: DataFrame):
return ('Made {:6d} buys. Average profit {: 5.2f}%. '
'Total profit was {: 7.3f}. Average duration {:5.1f} mins.').format(
len(results.index),
results.profit.mean() * 100.0,
results.profit.sum(),
results.profit_percent.mean() * 100.0,
results.profit_BTC.sum(),
results.duration.mean() * 5,
)
)
def buy_strategy_generator(params):
@ -232,7 +232,8 @@ def start(args):
logger.info('Using config: %s ...', args.config)
config = load_config(args.config)
pairs = config['exchange']['pair_whitelist']
PROCESSED = optimize.preprocess(optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval))
PROCESSED = optimize.preprocess(optimize.load_data(
pairs=pairs, ticker_interval=args.ticker_interval))
if args.mongodb:
logger.info('Using mongodb ...')

19
freqtrade/persistence.py
View File

@ -121,7 +121,9 @@ class Trade(_DECL_BASE):
self
)
def calc_open_trade_price(self, fee: Optional[float] = None) -> float:
def calc_open_trade_price(
self,
fee: Optional[float] = None) -> float:
"""
Calculate the open_rate in BTC
:param fee: fee to use on the open rate (optional).
@ -134,7 +136,10 @@ class Trade(_DECL_BASE):
fees = buy_trade * Decimal(fee or self.fee)
return float(buy_trade + fees)
def calc_close_trade_price(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float:
def calc_close_trade_price(
self,
rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
"""
Calculate the close_rate in BTC
:param fee: fee to use on the close rate (optional).
@ -152,7 +157,10 @@ class Trade(_DECL_BASE):
fees = sell_trade * Decimal(fee or self.fee)
return float(sell_trade - fees)
def calc_profit(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float:
def calc_profit(
self,
rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
"""
Calculate the profit in BTC between Close and Open trade
:param fee: fee to use on the close rate (optional).
@ -168,7 +176,10 @@ class Trade(_DECL_BASE):
)
return float("{0:.8f}".format(close_trade_price - open_trade_price))
def calc_profit_percent(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float:
def calc_profit_percent(
self,
rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
"""
Calculates the profit in percentage (including fee).
:param rate: rate to compare with (optional).

4
freqtrade/rpc/telegram.py
View File

@ -232,8 +232,8 @@ def _daily(bot: Bot, update: Update) -> None:
for day in range(0, timescale):
# need to query between day+1 and day-1
nextdate = date.fromordinal(today-day+1)
prevdate = date.fromordinal(today-day-1)
nextdate = date.fromordinal(today - day + 1)
prevdate = date.fromordinal(today - day - 1)
trades = Trade.query \
.filter(Trade.is_open.is_(False)) \
.filter(between(Trade.close_date, prevdate, nextdate)) \

9
freqtrade/tests/conftest.py
View File

@ -66,6 +66,7 @@ def ticker():
'last': 0.00001098,
})
@pytest.fixture
def ticker_sell_up():
return MagicMock(return_value={
@ -74,6 +75,7 @@ def ticker_sell_up():
'last': 0.00001172,
})
@pytest.fixture
def ticker_sell_down():
return MagicMock(return_value={
@ -82,6 +84,7 @@ def ticker_sell_down():
'last': 0.00001044,
})
@pytest.fixture
def health():
return MagicMock(return_value=[{
@ -143,7 +146,7 @@ def limit_sell_order():
@pytest.fixture
def ticker_history():
return [
{
{
"O": 8.794e-05,
"H": 8.948e-05,
"L": 8.794e-05,
@ -152,7 +155,7 @@ def ticker_history():
"T": "2017-11-26T08:50:00",
"BV": 0.0877869
},
{
{
"O": 8.88e-05,
"H": 8.942e-05,
"L": 8.88e-05,
@ -161,7 +164,7 @@ def ticker_history():
"T": "2017-11-26T08:55:00",
"BV": 0.05874751
},
{
{
"O": 8.891e-05,
"H": 8.893e-05,
"L": 8.875e-05,

11
freqtrade/tests/test_optimize_backtesting.py
View File

@ -1,15 +1,11 @@
# pragma pylint: disable=missing-docstring,W0212
from unittest.mock import MagicMock
from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex
from freqtrade.optimize.backtesting import backtest
from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata
import os
import pytest
def test_backtest(default_conf, mocker):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
@ -30,6 +26,7 @@ def test_1min_ticker_interval(default_conf, mocker):
results = backtest(default_conf, optimize.preprocess(data), 1, True)
assert len(results) > 0
def test_backtest_with_new_pair(default_conf, ticker_history, mocker):
mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history)
mocker.patch.dict('freqtrade.main._CONF', default_conf)
@ -59,7 +56,7 @@ def test_download_pairs(default_conf, ticker_history, mocker):
file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json'
file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json'
assert download_pairs(pairs = ['BTC-MEME', 'BTC-CFI']) is True
assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True
assert os.path.isfile(file1_1) is True
assert os.path.isfile(file1_5) is True
@ -87,7 +84,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
# Download a 1 min ticker file
file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json'
download_backtesting_testdata(pair = "BTC-XEL", interval = 1)
download_backtesting_testdata(pair="BTC-XEL", interval=1)
assert os.path.isfile(file1) is True
if os.path.isfile(file1):
@ -95,7 +92,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
# Download a 5 min ticker file
file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json'
download_backtesting_testdata(pair = "BTC-STORJ", interval = 5)
download_backtesting_testdata(pair="BTC-STORJ", interval=5)
assert os.path.isfile(file2) is True
if os.path.isfile(file2):

18
freqtrade/tests/test_persistence.py
View File

@ -10,17 +10,23 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order):
On this test we will buy and sell a crypto currency.
Buy
- Buy: 90.99181073 Crypto at 0.00001099 BTC (90.99181073*0.00001099 = 0.0009999 BTC)
- Buy: 90.99181073 Crypto at 0.00001099 BTC
(90.99181073*0.00001099 = 0.0009999 BTC)
- Buying fee: 0.25%
- Total cost of buy trade: 0.001002500 BTC ((90.99181073*0.00001099) + ((90.99181073*0.00001099)*0.0025))
- Total cost of buy trade: 0.001002500 BTC
((90.99181073*0.00001099) + ((90.99181073*0.00001099)*0.0025))
Sell
- Sell: 90.99181073 Crypto at 0.00001173 BTC (90.99181073*0.00001173 = 0,00106733394 BTC)
- Sell: 90.99181073 Crypto at 0.00001173 BTC
(90.99181073*0.00001173 = 0,00106733394 BTC)
- Selling fee: 0.25%
- Total cost of sell trade: 0.001064666 BTC ((90.99181073*0.00001173) - ((90.99181073*0.00001173)*0.0025))
- Total cost of sell trade: 0.001064666 BTC
((90.99181073*0.00001173) - ((90.99181073*0.00001173)*0.0025))
Profit/Loss: +0.000062166 BTC (Sell:0.001064666 - Buy:0.001002500)
Profit/Loss percentage: 0.0620 ((0.001064666/0.001002500)-1 = 6.20%)
Profit/Loss: +0.000062166 BTC
(Sell:0.001064666 - Buy:0.001002500)
Profit/Loss percentage: 0.0620
((0.001064666/0.001002500)-1 = 6.20%)
:param limit_buy_order:
:param limit_sell_order:

10
freqtrade/tests/test_rpc_telegram.py
View File

@ -1,6 +1,6 @@
# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103
import re
from datetime import datetime, date
from datetime import datetime
from random import randint
from unittest.mock import MagicMock
@ -151,7 +151,8 @@ def test_status_table_handle(default_conf, update, ticker, mocker):
assert msg_mock.call_count == 1
def test_profit_handle(default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker):
def test_profit_handle(
default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True)
msg_mock = MagicMock()
@ -246,7 +247,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m
# Create some test data
create_trade(0.001)
## Decrease the price and sell it
# Decrease the price and sell it
mocker.patch.multiple('freqtrade.main.exchange',
validate_pairs=MagicMock(),
get_ticker=ticker_sell_down)
@ -383,7 +384,6 @@ def test_performance_handle(
assert '<code>BTC_ETH\t6.20%</code>' in msg_mock.call_args_list[0][0][0]
def test_daily_handle(
default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
@ -419,7 +419,7 @@ def test_daily_handle(
assert msg_mock.call_count == 1
assert 'Daily' in msg_mock.call_args_list[0][0][0]
assert str(datetime.utcnow().date()) + ' 0.00006217 BTC' in msg_mock.call_args_list[0][0][0]
# Try invalid data
msg_mock.reset_mock()
update_state(State.RUNNING)

8
install_ta-lib.sh Executable file
View File

@ -0,0 +1,8 @@
if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then
curl -O http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
tar zxvf ta-lib-0.4.0-src.tar.gz
cd ta-lib && ./configure && make && sudo make install && cd ..
else
echo "TA-lib already installed, skipping download and build."
cd ta-lib && sudo make install && cd ..
fi

2
scripts/plot_dataframe.py
View File

@ -3,7 +3,6 @@
import matplotlib # Install PYQT5 manually if you want to test this helper function
matplotlib.use("Qt5Agg")
import matplotlib.pyplot as plt
from freqtrade import exchange, analyze
@ -52,4 +51,3 @@ def plot_analyzed_dataframe(pair: str) -> None:
if __name__ == '__main__':
plot_analyzed_dataframe('BTC_ETH')