diff --git a/.travis.yml b/.travis.yml index e89bdbbf1..a00c03330 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +sudo: true os: - linux language: python @@ -11,9 +11,7 @@ addons: - libdw-dev - binutils-dev install: -- wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz -- tar zxvf ta-lib-0.4.0-src.tar.gz -- cd ta-lib && ./configure && sudo make && sudo make install && cd .. +- ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install flake8 coveralls - pip install -r requirements.txt @@ -27,11 +25,13 @@ jobs: - script: - cp config.json.example config.json - python freqtrade/main.py hyperopt -e 5 + - script: flake8 freqtrade after_success: -- flake8 freqtrade && coveralls +- coveralls notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= cache: directories: - - $HOME/.cache/pip \ No newline at end of file + - $HOME/.cache/pip + - ta-lib \ No newline at end of file diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 841cad57e..ca7fce262 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -50,8 +50,8 @@ class Bittrex(Exchange): @property def fee(self) -> float: - # See https://bittrex.com/fees - return 0.0025 #0.25% + # 0.25 %: See https://bittrex.com/fees + return 0.0025 def buy(self, pair: str, rate: float, amount: float) -> str: data = _API.buy_limit(pair.replace('_', '-'), amount, rate) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 1dcd33842..b01fd9fe9 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -168,8 +168,8 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None: ) backtesting_cmd.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from Bittrex. Use it if you want to \ - run your backtesting with up-to-date data.', + help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \ + Use it if you want to run your backtesting with up-to-date data.', action='store_true', dest='refresh_pairs', ) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index dccc18093..12425cda2 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -13,7 +13,8 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe logger = logging.getLogger(__name__) -def load_data(pairs: List[str], ticker_interval: int = 5, refresh_pairs: Optional[bool] = False) -> Dict[str, List]: +def load_data(pairs: List[str], ticker_interval: int = 5, + refresh_pairs: Optional[bool] = False) -> Dict[str, List]: """ Loads ticker history data for the given parameters :param ticker_interval: ticker interval in minutes @@ -61,10 +62,10 @@ def download_pairs(pairs: List[str]) -> bool: """For each pairs passed in parameters, download 1 and 5 ticker intervals""" for pair in pairs: try: - for interval in [1,5]: + for interval in [1, 5]: download_backtesting_testdata(pair=pair, interval=interval) except BaseException: - logger.info('Impossible to download the pair: "{pair}", Interval: {interval} min'.format( + logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format( pair=pair, interval=interval, )) @@ -103,7 +104,7 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool: logger.debug("Current Start: None") logger.debug("Current End: None") - new_data = get_ticker_history(pair = pair, tick_interval = int(interval)) + new_data = get_ticker_history(pair=pair, tick_interval=int(interval)) for row in new_data: if row not in data: data.append(row) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 52fdce1cc..5761c9507 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -48,8 +48,8 @@ def generate_text_table( tabular_data.append([ pair, len(result.index), - '{:.2f}%'.format(result.profit.mean() * 100.0), - '{:.08f} {}'.format(result.profit.sum(), stake_currency), + '{:.2f}%'.format(result.profit_percent.mean() * 100.0), + '{:.08f} {}'.format(result.profit_BTC.sum(), stake_currency), '{:.2f}'.format(result.duration.mean() * ticker_interval), ]) @@ -57,8 +57,8 @@ def generate_text_table( tabular_data.append([ 'TOTAL', len(results.index), - '{:.2f}%'.format(results.profit.mean() * 100.0), - '{:.08f} {}'.format(results.profit.sum(), stake_currency), + '{:.2f}%'.format(results.profit_percent.mean() * 100.0), + '{:.08f} {}'.format(results.profit_BTC.sum(), stake_currency), '{:.2f}'.format(results.duration.mean() * ticker_interval), ]) return tabulate(tabular_data, headers=headers) @@ -98,7 +98,8 @@ def backtest(config: Dict, processed: Dict[str, DataFrame], trade = Trade( open_rate=row.close, open_date=row.date, - amount=config['stake_amount'], + stake_amount=config['stake_amount'], + amount=config['stake_amount'] / row.open, fee=exchange.get_fee() ) @@ -109,12 +110,20 @@ def backtest(config: Dict, processed: Dict[str, DataFrame], trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1: - current_profit = trade.calc_profit_percent(row2.close) + current_profit_percent = trade.calc_profit_percent(rate=row2.close) + current_profit_BTC = trade.calc_profit(rate=row2.close) lock_pair_until = row2.Index - trades.append((pair, current_profit, row2.Index - row.Index)) + trades.append( + ( + pair, + current_profit_percent, + current_profit_BTC, + row2.Index - row.Index + ) + ) break - labels = ['currency', 'profit', 'duration'] + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) @@ -140,7 +149,8 @@ def start(args): data[pair] = exchange.get_ticker_history(pair, args.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') - data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs) + data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, + refresh_pairs=args.refresh_pairs) logger.info('Using stake_currency: %s ...', config['stake_currency']) logger.info('Using stake_amount: %s ...', config['stake_amount']) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8603f7d8c..249edf0be 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -131,7 +131,7 @@ def optimizer(params): result = format_results(results) - total_profit = results.profit.sum() * 1000 + total_profit = results.profit_percent.sum() * 1000 trade_count = len(results.index) trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) @@ -144,13 +144,13 @@ def optimizer(params): 'total_profit': total_profit, 'trade_loss': trade_loss, 'profit_loss': profit_loss, - 'avg_profit': results.profit.mean() * 100.0, + 'avg_profit': results.profit_percent.mean() * 100.0, 'avg_duration': results.duration.mean() * 5, 'current_tries': _CURRENT_TRIES, 'total_tries': TOTAL_TRIES, 'result': result, 'results': results - } + } # logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result)) log_results(result_data) @@ -166,10 +166,10 @@ def format_results(results: DataFrame): return ('Made {:6d} buys. Average profit {: 5.2f}%. ' 'Total profit was {: 7.3f}. Average duration {:5.1f} mins.').format( len(results.index), - results.profit.mean() * 100.0, - results.profit.sum(), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), results.duration.mean() * 5, - ) + ) def buy_strategy_generator(params): @@ -232,7 +232,8 @@ def start(args): logger.info('Using config: %s ...', args.config) config = load_config(args.config) pairs = config['exchange']['pair_whitelist'] - PROCESSED = optimize.preprocess(optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval)) + PROCESSED = optimize.preprocess(optimize.load_data( + pairs=pairs, ticker_interval=args.ticker_interval)) if args.mongodb: logger.info('Using mongodb ...') diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ed2cd2029..9825299b7 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -121,7 +121,9 @@ class Trade(_DECL_BASE): self ) - def calc_open_trade_price(self, fee: Optional[float] = None) -> float: + def calc_open_trade_price( + self, + fee: Optional[float] = None) -> float: """ Calculate the open_rate in BTC :param fee: fee to use on the open rate (optional). @@ -134,7 +136,10 @@ class Trade(_DECL_BASE): fees = buy_trade * Decimal(fee or self.fee) return float(buy_trade + fees) - def calc_close_trade_price(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: + def calc_close_trade_price( + self, + rate: Optional[float] = None, + fee: Optional[float] = None) -> float: """ Calculate the close_rate in BTC :param fee: fee to use on the close rate (optional). @@ -152,7 +157,10 @@ class Trade(_DECL_BASE): fees = sell_trade * Decimal(fee or self.fee) return float(sell_trade - fees) - def calc_profit(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: + def calc_profit( + self, + rate: Optional[float] = None, + fee: Optional[float] = None) -> float: """ Calculate the profit in BTC between Close and Open trade :param fee: fee to use on the close rate (optional). @@ -168,7 +176,10 @@ class Trade(_DECL_BASE): ) return float("{0:.8f}".format(close_trade_price - open_trade_price)) - def calc_profit_percent(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: + def calc_profit_percent( + self, + rate: Optional[float] = None, + fee: Optional[float] = None) -> float: """ Calculates the profit in percentage (including fee). :param rate: rate to compare with (optional). diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index b2e2d85dc..e4069ebe8 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -232,8 +232,8 @@ def _daily(bot: Bot, update: Update) -> None: for day in range(0, timescale): # need to query between day+1 and day-1 - nextdate = date.fromordinal(today-day+1) - prevdate = date.fromordinal(today-day-1) + nextdate = date.fromordinal(today - day + 1) + prevdate = date.fromordinal(today - day - 1) trades = Trade.query \ .filter(Trade.is_open.is_(False)) \ .filter(between(Trade.close_date, prevdate, nextdate)) \ diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 5b1d98ef9..b034b8c9f 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -66,6 +66,7 @@ def ticker(): 'last': 0.00001098, }) + @pytest.fixture def ticker_sell_up(): return MagicMock(return_value={ @@ -74,6 +75,7 @@ def ticker_sell_up(): 'last': 0.00001172, }) + @pytest.fixture def ticker_sell_down(): return MagicMock(return_value={ @@ -82,6 +84,7 @@ def ticker_sell_down(): 'last': 0.00001044, }) + @pytest.fixture def health(): return MagicMock(return_value=[{ @@ -143,7 +146,7 @@ def limit_sell_order(): @pytest.fixture def ticker_history(): return [ - { + { "O": 8.794e-05, "H": 8.948e-05, "L": 8.794e-05, @@ -152,7 +155,7 @@ def ticker_history(): "T": "2017-11-26T08:50:00", "BV": 0.0877869 }, - { + { "O": 8.88e-05, "H": 8.942e-05, "L": 8.88e-05, @@ -161,7 +164,7 @@ def ticker_history(): "T": "2017-11-26T08:55:00", "BV": 0.05874751 }, - { + { "O": 8.891e-05, "H": 8.893e-05, "L": 8.875e-05, diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py index c43bcede5..f35d21648 100644 --- a/freqtrade/tests/test_optimize_backtesting.py +++ b/freqtrade/tests/test_optimize_backtesting.py @@ -1,15 +1,11 @@ # pragma pylint: disable=missing-docstring,W0212 -from unittest.mock import MagicMock - from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.backtesting import backtest from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata import os -import pytest - def test_backtest(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -30,6 +26,7 @@ def test_1min_ticker_interval(default_conf, mocker): results = backtest(default_conf, optimize.preprocess(data), 1, True) assert len(results) > 0 + def test_backtest_with_new_pair(default_conf, ticker_history, mocker): mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -59,7 +56,7 @@ def test_download_pairs(default_conf, ticker_history, mocker): file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json' file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json' - assert download_pairs(pairs = ['BTC-MEME', 'BTC-CFI']) is True + assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True assert os.path.isfile(file1_1) is True assert os.path.isfile(file1_5) is True @@ -87,7 +84,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): # Download a 1 min ticker file file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' - download_backtesting_testdata(pair = "BTC-XEL", interval = 1) + download_backtesting_testdata(pair="BTC-XEL", interval=1) assert os.path.isfile(file1) is True if os.path.isfile(file1): @@ -95,7 +92,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): # Download a 5 min ticker file file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' - download_backtesting_testdata(pair = "BTC-STORJ", interval = 5) + download_backtesting_testdata(pair="BTC-STORJ", interval=5) assert os.path.isfile(file2) is True if os.path.isfile(file2): diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 6315cfcf1..f91a8efd0 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -10,17 +10,23 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order): On this test we will buy and sell a crypto currency. Buy - - Buy: 90.99181073 Crypto at 0.00001099 BTC (90.99181073*0.00001099 = 0.0009999 BTC) + - Buy: 90.99181073 Crypto at 0.00001099 BTC + (90.99181073*0.00001099 = 0.0009999 BTC) - Buying fee: 0.25% - - Total cost of buy trade: 0.001002500 BTC ((90.99181073*0.00001099) + ((90.99181073*0.00001099)*0.0025)) + - Total cost of buy trade: 0.001002500 BTC + ((90.99181073*0.00001099) + ((90.99181073*0.00001099)*0.0025)) Sell - - Sell: 90.99181073 Crypto at 0.00001173 BTC (90.99181073*0.00001173 = 0,00106733394 BTC) + - Sell: 90.99181073 Crypto at 0.00001173 BTC + (90.99181073*0.00001173 = 0,00106733394 BTC) - Selling fee: 0.25% - - Total cost of sell trade: 0.001064666 BTC ((90.99181073*0.00001173) - ((90.99181073*0.00001173)*0.0025)) + - Total cost of sell trade: 0.001064666 BTC + ((90.99181073*0.00001173) - ((90.99181073*0.00001173)*0.0025)) - Profit/Loss: +0.000062166 BTC (Sell:0.001064666 - Buy:0.001002500) - Profit/Loss percentage: 0.0620 ((0.001064666/0.001002500)-1 = 6.20%) + Profit/Loss: +0.000062166 BTC + (Sell:0.001064666 - Buy:0.001002500) + Profit/Loss percentage: 0.0620 + ((0.001064666/0.001002500)-1 = 6.20%) :param limit_buy_order: :param limit_sell_order: diff --git a/freqtrade/tests/test_rpc_telegram.py b/freqtrade/tests/test_rpc_telegram.py index cf5b7051b..3b6197f23 100644 --- a/freqtrade/tests/test_rpc_telegram.py +++ b/freqtrade/tests/test_rpc_telegram.py @@ -1,6 +1,6 @@ # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 import re -from datetime import datetime, date +from datetime import datetime from random import randint from unittest.mock import MagicMock @@ -151,7 +151,8 @@ def test_status_table_handle(default_conf, update, ticker, mocker): assert msg_mock.call_count == 1 -def test_profit_handle(default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): +def test_profit_handle( + default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) msg_mock = MagicMock() @@ -246,7 +247,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m # Create some test data create_trade(0.001) - ## Decrease the price and sell it + # Decrease the price and sell it mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker_sell_down) @@ -383,7 +384,6 @@ def test_performance_handle( assert 'BTC_ETH\t6.20%' in msg_mock.call_args_list[0][0][0] - def test_daily_handle( default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -419,7 +419,7 @@ def test_daily_handle( assert msg_mock.call_count == 1 assert 'Daily' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) + ' 0.00006217 BTC' in msg_mock.call_args_list[0][0][0] - + # Try invalid data msg_mock.reset_mock() update_state(State.RUNNING) diff --git a/install_ta-lib.sh b/install_ta-lib.sh new file mode 100755 index 000000000..4b1320d4a --- /dev/null +++ b/install_ta-lib.sh @@ -0,0 +1,8 @@ +if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then + curl -O http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + tar zxvf ta-lib-0.4.0-src.tar.gz + cd ta-lib && ./configure && make && sudo make install && cd .. +else + echo "TA-lib already installed, skipping download and build." + cd ta-lib && sudo make install && cd .. +fi diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 0890a650e..0d193726d 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -3,7 +3,6 @@ import matplotlib # Install PYQT5 manually if you want to test this helper function matplotlib.use("Qt5Agg") import matplotlib.pyplot as plt - from freqtrade import exchange, analyze @@ -52,4 +51,3 @@ def plot_analyzed_dataframe(pair: str) -> None: if __name__ == '__main__': plot_analyzed_dataframe('BTC_ETH') -