From 5f1b9943d1667b3e6148f21a4a6f500a9c834346 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 17 Dec 2017 11:55:34 +0200 Subject: [PATCH 01/64] add smoke tests to run a round of hyperopt and backtesting --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37bd480a0..c6438b544 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,14 @@ install: - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install flake8 coveralls - pip install -r requirements.txt -script: -- pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ +jobs: + include: + - stage: test + - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ + - stage: hyperopt + - python freqtrade/main.py hyperopt -e 5 + - stage: backtesting + - python freqtrade/main.py backtesting after_success: - flake8 freqtrade && coveralls notifications: From a68ca31684cfe9c73d95a231239603fdc4f43914 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 17 Dec 2017 12:01:08 +0200 Subject: [PATCH 02/64] add smoke test commands under script block --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c6438b544..24c5b2a4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,11 +20,14 @@ install: jobs: include: - stage: test - - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ + script: + - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - stage: hyperopt - - python freqtrade/main.py hyperopt -e 5 + script: + - python freqtrade/main.py hyperopt -e 5 - stage: backtesting - - python freqtrade/main.py backtesting + script: + - python freqtrade/main.py backtesting after_success: - flake8 freqtrade && coveralls notifications: From 6288adfefd78bd6f23bfcafe8ebc3bd44d02c177 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 17 Dec 2017 14:14:57 +0200 Subject: [PATCH 03/64] fix plotting broken by refactoring --- scripts/plot_dataframe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index eeabda007..0890a650e 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -16,7 +16,8 @@ def plot_analyzed_dataframe(pair: str) -> None: # Init Bittrex to use public API exchange._API = exchange.Bittrex({'key': '', 'secret': ''}) - dataframe = analyze.analyze_ticker(pair) + ticker = exchange.get_ticker_history(pair) + dataframe = analyze.analyze_ticker(ticker) dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close'] From 21a11f55891c177d2dcb357bc5f61a6dd73a0929 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 17 Dec 2017 14:45:31 +0200 Subject: [PATCH 04/64] run pytest, hyperopt and backtesting in parallel --- .travis.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 24c5b2a4c..1d9d14a80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,15 +19,9 @@ install: - pip install -r requirements.txt jobs: include: - - stage: test - script: - - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - - stage: hyperopt - script: - - python freqtrade/main.py hyperopt -e 5 - - stage: backtesting - script: - - python freqtrade/main.py backtesting + - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ + - script: python freqtrade/main.py hyperopt -e 5 + - script: python freqtrade/main.py backtesting after_success: - flake8 freqtrade && coveralls notifications: From e83e4909a04a39f988c72bfa4d0c565b12b78cba Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 17 Dec 2017 15:01:11 +0200 Subject: [PATCH 05/64] install freqtrade module for hyperopting --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1d9d14a80..3f52c0985 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ install: - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install flake8 coveralls - pip install -r requirements.txt +- pip install -e . jobs: include: - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ From fe0c26f536889f72a4be3798ba428f5a62f3df3b Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 17 Dec 2017 15:13:39 +0200 Subject: [PATCH 06/64] create config.json for hyperopt --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3f52c0985..192cbcf8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,10 @@ install: jobs: include: - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - - script: python freqtrade/main.py hyperopt -e 5 - script: python freqtrade/main.py backtesting + - script: + - cp config.json.example config.json + - python freqtrade/main.py hyperopt -e 5 after_success: - flake8 freqtrade && coveralls notifications: From d3947fc89317de0746f07284177b9db893cedc5f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 17 Dec 2017 15:19:35 +0200 Subject: [PATCH 07/64] create config.json for backtesting --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 192cbcf8e..65e48c0db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,9 @@ install: jobs: include: - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - - script: python freqtrade/main.py backtesting + - script: + - cp config.json.example config.json + - python freqtrade/main.py backtesting - script: - cp config.json.example config.json - python freqtrade/main.py hyperopt -e 5 From 642422d5c4d9ac44843ff6461ffe8d53492f57fc Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 17 Dec 2017 22:19:50 +0200 Subject: [PATCH 08/64] cache pip dependencies (#199) --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 65e48c0db..e89bdbbf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,3 +32,6 @@ after_success: notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= +cache: + directories: + - $HOME/.cache/pip \ No newline at end of file From e3941cde7e9ce966c1db61cd6df9ba5ef8efd6d1 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Dec 2017 07:15:14 +0200 Subject: [PATCH 09/64] move wgetting and building of talib to an sh file --- .travis.yml | 4 +--- install_ta-lib.sh | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100755 install_ta-lib.sh diff --git a/.travis.yml b/.travis.yml index e89bdbbf1..0e4c22263 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,7 @@ addons: - libdw-dev - binutils-dev install: -- wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz -- tar zxvf ta-lib-0.4.0-src.tar.gz -- cd ta-lib && ./configure && sudo make && sudo make install && cd .. +- ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install flake8 coveralls - pip install -r requirements.txt diff --git a/install_ta-lib.sh b/install_ta-lib.sh new file mode 100755 index 000000000..077c08bae --- /dev/null +++ b/install_ta-lib.sh @@ -0,0 +1,3 @@ +wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz +tar zxvf ta-lib-0.4.0-src.tar.gz +cd ta-lib && ./configure && sudo make && sudo make install && cd .. From d613d63fdcea629c42a5a5dead54aa365f08cdd2 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 17 Dec 2017 13:07:56 -0800 Subject: [PATCH 10/64] Fix the fee calculation --- freqtrade/exchange/bittrex.py | 2 +- freqtrade/main.py | 13 +-- freqtrade/optimize/backtesting.py | 4 +- freqtrade/persistence.py | 69 ++++++++++-- freqtrade/rpc/telegram.py | 42 ++++---- freqtrade/tests/conftest.py | 33 ++++-- freqtrade/tests/test_main.py | 27 ++--- freqtrade/tests/test_persistence.py | 155 +++++++++++++++++++++++++-- freqtrade/tests/test_rpc_telegram.py | 82 ++++++++++---- 9 files changed, 344 insertions(+), 83 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 1a0a035a9..841cad57e 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -51,7 +51,7 @@ class Bittrex(Exchange): @property def fee(self) -> float: # See https://bittrex.com/fees - return 0.0025 + return 0.0025 #0.25% def buy(self, pair: str, rate: float, amount: float) -> str: data = _API.buy_limit(pair.replace('_', '-'), amount, rate) diff --git a/freqtrade/main.py b/freqtrade/main.py index 941d6870a..212e9f0e9 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -118,13 +118,14 @@ def execute_sell(trade: Trade, limit: float) -> None: order_id = exchange.sell(str(trade.pair), limit, trade.amount) trade.open_order_id = order_id - fmt_exp_profit = round(trade.calc_profit(limit) * 100, 2) - rpc.send_msg('*{}:* Selling [{}]({}) with limit `{:.8f} (profit: ~{:.2f}%)`'.format( + fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) + rpc.send_msg('*{}:* Selling [{}]({}) with limit `{:.8f} (profit: ~{:.2f}%, {:.8f})`'.format( trade.exchange, trade.pair.replace('_', '/'), exchange.get_pair_detail_url(trade.pair), limit, - fmt_exp_profit + fmt_exp_profit, + trade.calc_profit(rate=limit), )) Trade.session.flush() @@ -134,7 +135,7 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) - Based an earlier trade and current price and ROI configuration, decides whether bot should sell :return True if bot should sell at current rate """ - current_profit = trade.calc_profit(current_rate) + current_profit = trade.calc_profit_percent(current_rate) if 'stoploss' in _CONF and current_profit < float(_CONF['stoploss']): logger.debug('Stop loss hit.') return True @@ -145,7 +146,7 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) - if time_diff > float(duration) and current_profit > threshold: return True - logger.debug('Threshold not reached. (cur_profit: %1.2f%%)', current_profit * 100.0) + logger.debug('Threshold not reached. (cur_profit: %1.2f%%)', float(current_profit) * 100.0) return False @@ -233,7 +234,7 @@ def create_trade(stake_amount: float) -> bool: pair=pair, stake_amount=stake_amount, amount=amount, - fee=exchange.get_fee() * 2, + fee=exchange.get_fee(), open_rate=buy_limit, open_date=datetime.utcnow(), exchange=exchange.get_name().upper(), diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index dd7bf160c..52fdce1cc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -99,7 +99,7 @@ def backtest(config: Dict, processed: Dict[str, DataFrame], open_rate=row.close, open_date=row.date, amount=config['stake_amount'], - fee=exchange.get_fee() * 2 + fee=exchange.get_fee() ) # calculate win/lose forwards from buy point @@ -109,7 +109,7 @@ def backtest(config: Dict, processed: Dict[str, DataFrame], trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1: - current_profit = trade.calc_profit(row2.close) + current_profit = trade.calc_profit_percent(row2.close) lock_pair_until = row2.Index trades.append((pair, current_profit, row2.Index - row.Index)) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ba0ad1785..ed2cd2029 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -92,10 +92,12 @@ class Trade(_DECL_BASE): return logger.info('Updating trade (id=%d) ...', self.id) + + getcontext().prec = 8 # Bittrex do not go above 8 decimal if order['type'] == 'LIMIT_BUY': # Update open rate and actual amount - self.open_rate = order['rate'] - self.amount = order['amount'] + self.open_rate = Decimal(order['rate']) + self.amount = Decimal(order['amount']) logger.info('LIMIT_BUY has been fulfilled for %s.', self) self.open_order_id = None elif order['type'] == 'LIMIT_SELL': @@ -109,8 +111,8 @@ class Trade(_DECL_BASE): Sets close_rate to the given rate, calculates total profit and marks trade as closed """ - self.close_rate = rate - self.close_profit = self.calc_profit() + self.close_rate = Decimal(rate) + self.close_profit = self.calc_profit_percent() self.close_date = datetime.utcnow() self.is_open = False self.open_order_id = None @@ -119,7 +121,54 @@ class Trade(_DECL_BASE): self ) - def calc_profit(self, rate: Optional[float] = None) -> float: + def calc_open_trade_price(self, fee: Optional[float] = None) -> float: + """ + Calculate the open_rate in BTC + :param fee: fee to use on the open rate (optional). + If rate is not set self.fee will be used + :return: Price in BTC of the open trade + """ + getcontext().prec = 8 + + buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) + fees = buy_trade * Decimal(fee or self.fee) + return float(buy_trade + fees) + + def calc_close_trade_price(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: + """ + Calculate the close_rate in BTC + :param fee: fee to use on the close rate (optional). + If rate is not set self.fee will be used + :param rate: rate to compare with (optional). + If rate is not set self.close_rate will be used + :return: Price in BTC of the open trade + """ + getcontext().prec = 8 + + if rate is None and not self.close_rate: + return 0.0 + + sell_trade = (Decimal(self.amount) * Decimal(rate or self.close_rate)) + fees = sell_trade * Decimal(fee or self.fee) + return float(sell_trade - fees) + + def calc_profit(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: + """ + Calculate the profit in BTC between Close and Open trade + :param fee: fee to use on the close rate (optional). + If rate is not set self.fee will be used + :param rate: close rate to compare with (optional). + If rate is not set self.close_rate will be used + :return: profit in BTC as float + """ + open_trade_price = self.calc_open_trade_price() + close_trade_price = self.calc_close_trade_price( + rate=Decimal(rate or self.close_rate), + fee=Decimal(fee or self.fee) + ) + return float("{0:.8f}".format(close_trade_price - open_trade_price)) + + def calc_profit_percent(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ Calculates the profit in percentage (including fee). :param rate: rate to compare with (optional). @@ -127,5 +176,11 @@ class Trade(_DECL_BASE): :return: profit in percentage as float """ getcontext().prec = 8 - return float((Decimal(rate or self.close_rate) - Decimal(self.open_rate)) - / Decimal(self.open_rate) - Decimal(self.fee)) + + open_trade_price = self.calc_open_trade_price() + close_trade_price = self.calc_close_trade_price( + rate=Decimal(rate or self.close_rate), + fee=Decimal(fee or self.fee) + ) + + return float("{0:.8f}".format((close_trade_price / open_trade_price) - 1)) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a097e45f4..b2e2d85dc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,7 +1,7 @@ import logging import re -from datetime import timedelta, date from decimal import Decimal +from datetime import timedelta, date, datetime from typing import Callable, Any import arrow @@ -139,7 +139,7 @@ def _status(bot: Bot, update: Update) -> None: order = exchange.get_order(trade.open_order_id) # calculate profit and send message to user current_rate = exchange.get_ticker(trade.pair)['bid'] - current_profit = trade.calc_profit(current_rate) + current_profit = trade.calc_profit_percent(current_rate) fmt_close_profit = '{:.2f}%'.format( round(trade.close_profit * 100, 2) ) if trade.close_profit else None @@ -196,7 +196,7 @@ def _status_table(bot: Bot, update: Update) -> None: trade.id, trade.pair, shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}'.format(100 * trade.calc_profit(current_rate)) + '{:.2f}'.format(100 * trade.calc_profit_percent(current_rate)) ]) columns = ['ID', 'Pair', 'Since', 'Profit'] @@ -218,7 +218,7 @@ def _daily(bot: Bot, update: Update) -> None: :param update: message update :return: None """ - today = date.today().toordinal() + today = datetime.utcnow().toordinal() profit_days = {} try: @@ -234,9 +234,13 @@ def _daily(bot: Bot, update: Update) -> None: # need to query between day+1 and day-1 nextdate = date.fromordinal(today-day+1) prevdate = date.fromordinal(today-day-1) - trades = Trade.query.filter(between(Trade.close_date, prevdate, nextdate)).all() - curdayprofit = sum(trade.close_profit * trade.stake_amount for trade in trades) - profit_days[date.fromordinal(today-day)] = format(curdayprofit, '.8f') + trades = Trade.query \ + .filter(Trade.is_open.is_(False)) \ + .filter(between(Trade.close_date, prevdate, nextdate)) \ + .order_by(Trade.close_date)\ + .all() + curdayprofit = sum(trade.calc_profit() for trade in trades) + profit_days[date.fromordinal(today - day)] = format(curdayprofit, '.8f') stats = [[key, str(value) + ' BTC'] for key, value in profit_days.items()] stats = tabulate(stats, headers=['Day', 'Profit'], tablefmt='simple') @@ -257,9 +261,9 @@ def _profit(bot: Bot, update: Update) -> None: trades = Trade.query.order_by(Trade.id).all() profit_all_btc = [] - profit_all = [] + profit_all_percent = [] profit_btc_closed = [] - profit_closed = [] + profit_closed_percent = [] durations = [] for trade in trades: @@ -271,16 +275,16 @@ def _profit(bot: Bot, update: Update) -> None: durations.append((trade.close_date - trade.open_date).total_seconds()) if not trade.is_open: - profit = trade.close_profit - profit_btc_closed.append(Decimal(trade.close_rate) - Decimal(trade.open_rate)) - profit_closed.append(profit) + profit_percent = trade.calc_profit_percent() + profit_btc_closed.append(trade.calc_profit()) + profit_closed_percent.append(profit_percent) else: # Get current rate current_rate = exchange.get_ticker(trade.pair)['bid'] - profit = trade.calc_profit(current_rate) + profit_percent = trade.calc_profit_percent(rate=current_rate) - profit_all_btc.append(Decimal(trade.close_rate or current_rate) - Decimal(trade.open_rate)) - profit_all.append(profit) + profit_all_btc.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate))) + profit_all_percent.append(profit_percent) best_pair = Trade.session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ .filter(Trade.is_open.is_(False)) \ @@ -294,8 +298,8 @@ def _profit(bot: Bot, update: Update) -> None: bp_pair, bp_rate = best_pair markdown_msg = """ -*ROI Trade closed:* `{profit_closed_btc:.8f} BTC ({profit_closed:.2f}%)` -*ROI All trades:* `{profit_all_btc:.8f} BTC ({profit_all:.2f}%)` +*ROI Trade closed:* `{profit_closed_btc:.8f} BTC ({profit_closed_percent:.2f}%)` +*ROI All trades:* `{profit_all_btc:.8f} BTC ({profit_all_percent:.2f}%)` *Total Trade Count:* `{trade_count}` *First Trade opened:* `{first_trade_date}` *Latest Trade opened:* `{latest_trade_date}` @@ -303,9 +307,9 @@ def _profit(bot: Bot, update: Update) -> None: *Best Performing:* `{best_pair}: {best_rate:.2f}%` """.format( profit_closed_btc=round(sum(profit_btc_closed), 8), - profit_closed=round(sum(profit_closed) * 100, 2), + profit_closed_percent=round(sum(profit_closed_percent) * 100, 2), profit_all_btc=round(sum(profit_all_btc), 8), - profit_all=round(sum(profit_all) * 100, 2), + profit_all_percent=round(sum(profit_all_percent) * 100, 2), trade_count=len(trades), first_trade_date=arrow.get(trades[0].open_date).humanize(), latest_trade_date=arrow.get(trades[-1].open_date).humanize(), diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index c711f3953..5b1d98ef9 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -15,7 +15,7 @@ def default_conf(): configuration = { "max_open_trades": 1, "stake_currency": "BTC", - "stake_amount": 0.05, + "stake_amount": 0.001, "dry_run": True, "minimal_roi": { "40": 0.0, @@ -61,11 +61,26 @@ def update(): @pytest.fixture def ticker(): return MagicMock(return_value={ - 'bid': 0.07256061, - 'ask': 0.072661, - 'last': 0.07256061, + 'bid': 0.00001098, + 'ask': 0.00001099, + 'last': 0.00001098, }) +@pytest.fixture +def ticker_sell_up(): + return MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172, + }) + +@pytest.fixture +def ticker_sell_down(): + return MagicMock(return_value={ + 'bid': 0.00001044, + 'ask': 0.00001043, + 'last': 0.00001044, + }) @pytest.fixture def health(): @@ -104,8 +119,8 @@ def limit_buy_order(): 'type': 'LIMIT_BUY', 'pair': 'mocked', 'opened': datetime.utcnow(), - 'rate': 0.07256061, - 'amount': 206.43811673387373, + 'rate': 0.00001099, + 'amount': 90.99181073, 'remaining': 0.0, 'closed': datetime.utcnow(), } @@ -118,8 +133,8 @@ def limit_sell_order(): 'type': 'LIMIT_SELL', 'pair': 'mocked', 'opened': datetime.utcnow(), - 'rate': 0.0802134, - 'amount': 206.43811673387373, + 'rate': 0.00001173, + 'amount': 90.99181073, 'remaining': 0.0, 'closed': datetime.utcnow(), } @@ -155,4 +170,4 @@ def ticker_history(): "T": "2017-11-26T09:00:00", "BV": 0.7039405 } - ] \ No newline at end of file + ] diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index aeea5a1a1..d9b774a9d 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -40,8 +40,8 @@ def test_process_trade_creation(default_conf, ticker, health, mocker): assert trade.is_open assert trade.open_date is not None assert trade.exchange == Exchanges.BITTREX.name - assert trade.open_rate == 0.072661 - assert trade.amount == 0.6881270557795791 + assert trade.open_rate == 0.00001099 + assert trade.amount == 90.99181073703367 def test_process_exchange_failures(default_conf, ticker, health, mocker): @@ -115,11 +115,11 @@ def test_create_trade(default_conf, ticker, limit_buy_order, mocker): whitelist = copy.deepcopy(default_conf['exchange']['pair_whitelist']) init(default_conf, create_engine('sqlite://')) - create_trade(15.0) + create_trade(0.001) trade = Trade.query.first() assert trade is not None - assert trade.stake_amount == 15.0 + assert trade.stake_amount == 0.001 assert trade.is_open assert trade.open_date is not None assert trade.exchange == Exchanges.BITTREX.name @@ -127,8 +127,8 @@ def test_create_trade(default_conf, ticker, limit_buy_order, mocker): # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) - assert trade.open_rate == 0.07256061 - assert trade.amount == 206.43811673387373 + assert trade.open_rate == 0.00001099 + assert trade.amount == 90.99181073 assert whitelist == default_conf['exchange']['pair_whitelist'] @@ -186,14 +186,14 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.17256061, - 'ask': 0.172661, - 'last': 0.17256061 + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 }), buy=MagicMock(return_value='mocked_limit_buy'), sell=MagicMock(return_value='mocked_limit_sell')) init(default_conf, create_engine('sqlite://')) - create_trade(15.0) + create_trade(0.001) trade = Trade.query.first() assert trade @@ -207,8 +207,9 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): # Simulate fulfilled LIMIT_SELL order for trade trade.update(limit_sell_order) - assert trade.close_rate == 0.0802134 - assert trade.close_profit == 0.10046755 + assert trade.close_rate == 0.00001173 + assert trade.close_profit == 0.06201057 + assert trade.calc_profit() == 0.00006217 assert trade.close_date is not None @@ -223,7 +224,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo # Create trade and sell it init(default_conf, create_engine('sqlite://')) - create_trade(15.0) + create_trade(0.001) trade = Trade.query.first() assert trade diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index eeb89d831..6315cfcf1 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -5,11 +5,32 @@ from freqtrade.exchange import Exchanges from freqtrade.persistence import Trade -def test_update(limit_buy_order, limit_sell_order): +def test_update_with_bittrex(limit_buy_order, limit_sell_order): + """ + On this test we will buy and sell a crypto currency. + + Buy + - Buy: 90.99181073 Crypto at 0.00001099 BTC (90.99181073*0.00001099 = 0.0009999 BTC) + - Buying fee: 0.25% + - Total cost of buy trade: 0.001002500 BTC ((90.99181073*0.00001099) + ((90.99181073*0.00001099)*0.0025)) + + Sell + - Sell: 90.99181073 Crypto at 0.00001173 BTC (90.99181073*0.00001173 = 0,00106733394 BTC) + - Selling fee: 0.25% + - Total cost of sell trade: 0.001064666 BTC ((90.99181073*0.00001173) - ((90.99181073*0.00001173)*0.0025)) + + Profit/Loss: +0.000062166 BTC (Sell:0.001064666 - Buy:0.001002500) + Profit/Loss percentage: 0.0620 ((0.001064666/0.001002500)-1 = 6.20%) + + :param limit_buy_order: + :param limit_sell_order: + :return: + """ + trade = Trade( pair='BTC_ETH', - stake_amount=1.00, - fee=0.1, + stake_amount=0.001, + fee=0.0025, exchange=Exchanges.BITTREX, ) assert trade.open_order_id is None @@ -20,18 +41,40 @@ def test_update(limit_buy_order, limit_sell_order): trade.open_order_id = 'something' trade.update(limit_buy_order) assert trade.open_order_id is None - assert trade.open_rate == 0.07256061 + assert trade.open_rate == 0.00001099 assert trade.close_profit is None assert trade.close_date is None trade.open_order_id = 'something' trade.update(limit_sell_order) assert trade.open_order_id is None - assert trade.open_rate == 0.07256061 - assert trade.close_profit == 0.00546755 + assert trade.close_rate == 0.00001173 + assert trade.close_profit == 0.06201057 assert trade.close_date is not None +def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order): + trade = Trade( + pair='BTC_ETH', + stake_amount=0.001, + fee=0.0025, + exchange=Exchanges.BITTREX, + ) + + trade.open_order_id = 'something' + trade.update(limit_buy_order) + assert trade.calc_open_trade_price() == 0.001002500 + + trade.update(limit_sell_order) + assert trade.calc_close_trade_price() == 0.0010646656 + + # Profit in BTC + assert trade.calc_profit() == 0.00006217 + + # Profit in percent + assert trade.calc_profit_percent() == 0.06201057 + + def test_update_open_order(limit_buy_order): trade = Trade( pair='BTC_ETH', @@ -64,3 +107,103 @@ def test_update_invalid_order(limit_buy_order): limit_buy_order['type'] = 'invalid' with pytest.raises(ValueError, match=r'Unknown order type'): trade.update(limit_buy_order) + + +def test_calc_open_trade_price(limit_buy_order): + trade = Trade( + pair='BTC_ETH', + stake_amount=0.001, + fee=0.0025, + exchange=Exchanges.BITTREX, + ) + trade.open_order_id = 'open_trade' + trade.update(limit_buy_order) # Buy @ 0.00001099 + + # Get the open rate price with the standard fee rate + assert trade.calc_open_trade_price() == 0.001002500 + + # Get the open rate price with a custom fee rate + assert trade.calc_open_trade_price(fee=0.003) == 0.001003000 + + +def test_calc_close_trade_price(limit_buy_order, limit_sell_order): + trade = Trade( + pair='BTC_ETH', + stake_amount=0.001, + fee=0.0025, + exchange=Exchanges.BITTREX, + ) + trade.open_order_id = 'close_trade' + trade.update(limit_buy_order) # Buy @ 0.00001099 + + # Get the close rate price with a custom close rate and a regular fee rate + assert trade.calc_close_trade_price(rate=0.00001234) == 0.0011200318 + + # Get the close rate price with a custom close rate and a custom fee rate + assert trade.calc_close_trade_price(rate=0.00001234, fee=0.003) == 0.0011194704 + + # Test when we apply a Sell order, and ask price with a custom fee rate + trade.update(limit_sell_order) + assert trade.calc_close_trade_price(fee=0.005) == 0.0010619972 + + +def test_calc_profit(limit_buy_order, limit_sell_order): + trade = Trade( + pair='BTC_ETH', + stake_amount=0.001, + fee=0.0025, + exchange=Exchanges.BITTREX, + ) + trade.open_order_id = 'profit_percent' + trade.update(limit_buy_order) # Buy @ 0.00001099 + + # Custom closing rate and regular fee rate + # Higher than open rate + assert trade.calc_profit(rate=0.00001234) == 0.00011753 + # Lower than open rate + assert trade.calc_profit(rate=0.00000123) == -0.00089086 + + # Custom closing rate and custom fee rate + # Higher than open rate + assert trade.calc_profit(rate=0.00001234, fee=0.003) == 0.00011697 + # Lower than open rate + assert trade.calc_profit(rate=0.00000123, fee=0.003) == -0.00089092 + + # Only custom fee without sell order applied + with pytest.raises(TypeError): + trade.calc_profit(fee=0.003) + + # Test when we apply a Sell order. Sell higher than open rate @ 0.00001173 + trade.update(limit_sell_order) + assert trade.calc_profit() == 0.00006217 + + # Test with a custom fee rate on the close trade + assert trade.calc_profit(fee=0.003) == 0.00006163 + + +def test_calc_profit_percent(limit_buy_order, limit_sell_order): + trade = Trade( + pair='BTC_ETH', + stake_amount=0.001, + fee=0.0025, + exchange=Exchanges.BITTREX, + ) + trade.open_order_id = 'profit_percent' + trade.update(limit_buy_order) # Buy @ 0.00001099 + + # Get percent of profit with a custom rate (Higher than open rate) + assert trade.calc_profit_percent(rate=0.00001234) == 0.1172387 + + # Get percent of profit with a custom rate (Lower than open rate) + assert trade.calc_profit_percent(rate=0.00000123) == -0.88863827 + + # Only custom fee without sell order applied + with pytest.raises(TypeError): + trade.calc_profit_percent(fee=0.003) + + # Test when we apply a Sell order. Sell higher than open rate @ 0.00001173 + trade.update(limit_sell_order) + assert trade.calc_profit_percent() == 0.06201057 + + # Test with a custom fee rate on the close trade + assert trade.calc_profit_percent(fee=0.003) == 0.0614782 diff --git a/freqtrade/tests/test_rpc_telegram.py b/freqtrade/tests/test_rpc_telegram.py index afdef2692..cf5b7051b 100644 --- a/freqtrade/tests/test_rpc_telegram.py +++ b/freqtrade/tests/test_rpc_telegram.py @@ -102,7 +102,7 @@ def test_status_handle(default_conf, update, ticker, mocker): msg_mock.reset_mock() # Create some test data - create_trade(15.0) + create_trade(0.001) # Trigger status while we have a fulfilled order for the open trade _status(bot=MagicMock(), update=update) @@ -151,7 +151,7 @@ def test_status_table_handle(default_conf, update, ticker, mocker): assert msg_mock.call_count == 1 -def test_profit_handle(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): +def test_profit_handle(default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) msg_mock = MagicMock() @@ -171,7 +171,7 @@ def test_profit_handle(default_conf, update, ticker, limit_buy_order, limit_sell msg_mock.reset_mock() # Create some test data - create_trade(15.0) + create_trade(0.001) trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade @@ -182,7 +182,10 @@ def test_profit_handle(default_conf, update, ticker, limit_buy_order, limit_sell assert 'no closed trade' in msg_mock.call_args_list[-1][0][0] msg_mock.reset_mock() - # Simulate fulfilled LIMIT_SELL order for trade + # Update the ticker with a market going up + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -190,11 +193,12 @@ def test_profit_handle(default_conf, update, ticker, limit_buy_order, limit_sell _profit(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert '*ROI All trades:* `0.00765279 BTC (10.05%)`' in msg_mock.call_args_list[-1][0][0] - assert 'Best Performing:* `BTC_ETH: 10.05%`' in msg_mock.call_args_list[-1][0][0] + assert '*ROI Trade closed:* `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert '*ROI All trades:* `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert 'Best Performing:* `BTC_ETH: 6.20%`' in msg_mock.call_args_list[-1][0][0] -def test_forcesell_handle(default_conf, update, ticker, mocker): +def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) @@ -208,7 +212,44 @@ def test_forcesell_handle(default_conf, update, ticker, mocker): init(default_conf, create_engine('sqlite://')) # Create some test data - create_trade(15.0) + create_trade(0.001) + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up) + + update.message.text = '/forcesell 1' + _forcesell(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 2 + assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172 (profit: ~6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] + + +def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) + mocker.patch.multiple('freqtrade.rpc.telegram', + _CONF=default_conf, + init=MagicMock(), + send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker) + init(default_conf, create_engine('sqlite://')) + + # Create some test data + create_trade(0.001) + + ## Decrease the price and sell it + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down) trade = Trade.query.first() assert trade @@ -218,7 +259,7 @@ def test_forcesell_handle(default_conf, update, ticker, mocker): assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] - assert '0.07256061 (profit: ~-0.64%)' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044 (profit: ~-5.48%, -0.00005492)' in rpc_mock.call_args_list[-1][0][0] def test_exec_forcesell_open_orders(default_conf, ticker, mocker): @@ -260,7 +301,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, mocker): # Create some test data for _ in range(4): - create_trade(15.0) + create_trade(0.001) rpc_mock.reset_mock() update.message.text = '/forcesell all' @@ -268,7 +309,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, mocker): assert rpc_mock.call_count == 4 for args in rpc_mock.call_args_list: - assert '0.07256061 (profit: ~-0.64%)' in args[0][0] + assert '0.00001098 (profit: ~-0.59%, -0.00000591)' in args[0][0] def test_forcesell_handle_invalid(default_conf, update, mocker): @@ -323,7 +364,7 @@ def test_performance_handle( init(default_conf, create_engine('sqlite://')) # Create some test data - create_trade(15.0) + create_trade(0.001) trade = Trade.query.first() assert trade @@ -339,7 +380,8 @@ def test_performance_handle( _performance(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'Performance' in msg_mock.call_args_list[0][0][0] - assert 'BTC_ETH\t10.05%' in msg_mock.call_args_list[0][0][0] + assert 'BTC_ETH\t6.20%' in msg_mock.call_args_list[0][0][0] + def test_daily_handle( @@ -358,7 +400,7 @@ def test_daily_handle( init(default_conf, create_engine('sqlite://')) # Create some test data - create_trade(15.0) + create_trade(0.001) trade = Trade.query.first() assert trade @@ -371,14 +413,14 @@ def test_daily_handle( trade.close_date = datetime.utcnow() trade.is_open = False - # try valid data - update.message.text = '/daily 7' + # Try valid data + update.message.text = '/daily 2' _daily(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'Daily' in msg_mock.call_args_list[0][0][0] - assert str(date.today()) + ' 1.50701325 BTC' in msg_mock.call_args_list[0][0][0] - - # try invalid data + assert str(datetime.utcnow().date()) + ' 0.00006217 BTC' in msg_mock.call_args_list[0][0][0] + + # Try invalid data msg_mock.reset_mock() update_state(State.RUNNING) update.message.text = '/daily -2' @@ -409,7 +451,7 @@ def test_count_handle(default_conf, update, ticker, mocker): update_state(State.RUNNING) # Create some test data - create_trade(15.0) + create_trade(0.001) msg_mock.reset_mock() _count(bot=MagicMock(), update=update) From 4c0a316e3e5908592e05240815177220d69a9194 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Dec 2017 09:14:40 +0200 Subject: [PATCH 11/64] enable sudo for installing talib --- .travis.yml | 2 +- install_ta-lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e4c22263..cc412d83c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +sudo: true os: - linux language: python diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 077c08bae..c37147747 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,3 +1,3 @@ wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar zxvf ta-lib-0.4.0-src.tar.gz -cd ta-lib && ./configure && sudo make && sudo make install && cd .. +cd ta-lib && ./configure && make && sudo make install && cd .. From e5f8c1e75d175b7c0356aca40fdddddc8988e4e1 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Dec 2017 09:29:17 +0200 Subject: [PATCH 12/64] cache ta-lib folder, skip build if cache exists --- .travis.yml | 3 ++- install_ta-lib.sh | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc412d83c..55f40d1c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,4 +32,5 @@ notifications: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= cache: directories: - - $HOME/.cache/pip \ No newline at end of file + - $HOME/.cache/pip + - ta-lib \ No newline at end of file diff --git a/install_ta-lib.sh b/install_ta-lib.sh index c37147747..6fb70a8c3 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,3 +1,8 @@ -wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz -tar zxvf ta-lib-0.4.0-src.tar.gz -cd ta-lib && ./configure && make && sudo make install && cd .. +if [ ! -d "ta-lib" ]; then + wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + tar zxvf ta-lib-0.4.0-src.tar.gz + cd ta-lib && ./configure && make && sudo make install && cd .. +else + echo "TA-lib already installed, skipping download and build." + cd ta-lib && sudo make install && cd .. +fi From 92f6db5bd7dd7233af547d512dbbe45dc4536076 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Dec 2017 09:36:29 +0200 Subject: [PATCH 13/64] fix checking for cached ta-lib --- install_ta-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 6fb70a8c3..10118a819 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,4 +1,4 @@ -if [ ! -d "ta-lib" ]; then +if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib && ./configure && make && sudo make install && cd .. From 98650acca0a2a0814ffd90f738a387ad340ca884 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Mon, 18 Dec 2017 10:26:48 +0200 Subject: [PATCH 14/64] use curl instead of wget (see travis-ci/issues/5059) --- install_ta-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 10118a819..4b1320d4a 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,5 +1,5 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + curl -O http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib && ./configure && make && sudo make install && cd .. else From c8fb6c46616d6a83770c81029800f766dcd72b70 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Dec 2017 18:36:00 +0200 Subject: [PATCH 15/64] More lint fixes (#198) * autopep fixes * remove unused imports * fix plot_dataframe.py lint warnings * make pep8 error fails the build * two more line breakings * matplotlib.use() must be called before pyplot import --- .travis.yml | 3 ++- freqtrade/misc.py | 4 ++-- freqtrade/optimize/__init__.py | 9 +++++---- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/optimize/hyperopt.py | 7 ++++--- freqtrade/rpc/telegram.py | 6 +++--- freqtrade/tests/conftest.py | 8 ++++---- freqtrade/tests/test_optimize_backtesting.py | 11 ++++------- scripts/plot_dataframe.py | 2 -- 9 files changed, 26 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55f40d1c8..a00c03330 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,8 +25,9 @@ jobs: - script: - cp config.json.example config.json - python freqtrade/main.py hyperopt -e 5 + - script: flake8 freqtrade after_success: -- flake8 freqtrade && coveralls +- coveralls notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 1dcd33842..b01fd9fe9 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -168,8 +168,8 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None: ) backtesting_cmd.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from Bittrex. Use it if you want to \ - run your backtesting with up-to-date data.', + help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \ + Use it if you want to run your backtesting with up-to-date data.', action='store_true', dest='refresh_pairs', ) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index dccc18093..12425cda2 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -13,7 +13,8 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe logger = logging.getLogger(__name__) -def load_data(pairs: List[str], ticker_interval: int = 5, refresh_pairs: Optional[bool] = False) -> Dict[str, List]: +def load_data(pairs: List[str], ticker_interval: int = 5, + refresh_pairs: Optional[bool] = False) -> Dict[str, List]: """ Loads ticker history data for the given parameters :param ticker_interval: ticker interval in minutes @@ -61,10 +62,10 @@ def download_pairs(pairs: List[str]) -> bool: """For each pairs passed in parameters, download 1 and 5 ticker intervals""" for pair in pairs: try: - for interval in [1,5]: + for interval in [1, 5]: download_backtesting_testdata(pair=pair, interval=interval) except BaseException: - logger.info('Impossible to download the pair: "{pair}", Interval: {interval} min'.format( + logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format( pair=pair, interval=interval, )) @@ -103,7 +104,7 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool: logger.debug("Current Start: None") logger.debug("Current End: None") - new_data = get_ticker_history(pair = pair, tick_interval = int(interval)) + new_data = get_ticker_history(pair=pair, tick_interval=int(interval)) for row in new_data: if row not in data: data.append(row) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index dd7bf160c..c281d439e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -140,7 +140,8 @@ def start(args): data[pair] = exchange.get_ticker_history(pair, args.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') - data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs) + data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, + refresh_pairs=args.refresh_pairs) logger.info('Using stake_currency: %s ...', config['stake_currency']) logger.info('Using stake_amount: %s ...', config['stake_amount']) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8603f7d8c..07e988a91 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -150,7 +150,7 @@ def optimizer(params): 'total_tries': TOTAL_TRIES, 'result': result, 'results': results - } + } # logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result)) log_results(result_data) @@ -169,7 +169,7 @@ def format_results(results: DataFrame): results.profit.mean() * 100.0, results.profit.sum(), results.duration.mean() * 5, - ) + ) def buy_strategy_generator(params): @@ -232,7 +232,8 @@ def start(args): logger.info('Using config: %s ...', args.config) config = load_config(args.config) pairs = config['exchange']['pair_whitelist'] - PROCESSED = optimize.preprocess(optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval)) + PROCESSED = optimize.preprocess(optimize.load_data( + pairs=pairs, ticker_interval=args.ticker_interval)) if args.mongodb: logger.info('Using mongodb ...') diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a097e45f4..f6d6ce234 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -232,11 +232,11 @@ def _daily(bot: Bot, update: Update) -> None: for day in range(0, timescale): # need to query between day+1 and day-1 - nextdate = date.fromordinal(today-day+1) - prevdate = date.fromordinal(today-day-1) + nextdate = date.fromordinal(today - day + 1) + prevdate = date.fromordinal(today - day - 1) trades = Trade.query.filter(between(Trade.close_date, prevdate, nextdate)).all() curdayprofit = sum(trade.close_profit * trade.stake_amount for trade in trades) - profit_days[date.fromordinal(today-day)] = format(curdayprofit, '.8f') + profit_days[date.fromordinal(today - day)] = format(curdayprofit, '.8f') stats = [[key, str(value) + ' BTC'] for key, value in profit_days.items()] stats = tabulate(stats, headers=['Day', 'Profit'], tablefmt='simple') diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index c711f3953..7e7908748 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -128,7 +128,7 @@ def limit_sell_order(): @pytest.fixture def ticker_history(): return [ - { + { "O": 8.794e-05, "H": 8.948e-05, "L": 8.794e-05, @@ -137,7 +137,7 @@ def ticker_history(): "T": "2017-11-26T08:50:00", "BV": 0.0877869 }, - { + { "O": 8.88e-05, "H": 8.942e-05, "L": 8.88e-05, @@ -146,7 +146,7 @@ def ticker_history(): "T": "2017-11-26T08:55:00", "BV": 0.05874751 }, - { + { "O": 8.891e-05, "H": 8.893e-05, "L": 8.875e-05, @@ -155,4 +155,4 @@ def ticker_history(): "T": "2017-11-26T09:00:00", "BV": 0.7039405 } - ] \ No newline at end of file + ] diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py index c43bcede5..f35d21648 100644 --- a/freqtrade/tests/test_optimize_backtesting.py +++ b/freqtrade/tests/test_optimize_backtesting.py @@ -1,15 +1,11 @@ # pragma pylint: disable=missing-docstring,W0212 -from unittest.mock import MagicMock - from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.backtesting import backtest from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata import os -import pytest - def test_backtest(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -30,6 +26,7 @@ def test_1min_ticker_interval(default_conf, mocker): results = backtest(default_conf, optimize.preprocess(data), 1, True) assert len(results) > 0 + def test_backtest_with_new_pair(default_conf, ticker_history, mocker): mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -59,7 +56,7 @@ def test_download_pairs(default_conf, ticker_history, mocker): file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json' file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json' - assert download_pairs(pairs = ['BTC-MEME', 'BTC-CFI']) is True + assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True assert os.path.isfile(file1_1) is True assert os.path.isfile(file1_5) is True @@ -87,7 +84,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): # Download a 1 min ticker file file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' - download_backtesting_testdata(pair = "BTC-XEL", interval = 1) + download_backtesting_testdata(pair="BTC-XEL", interval=1) assert os.path.isfile(file1) is True if os.path.isfile(file1): @@ -95,7 +92,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): # Download a 5 min ticker file file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' - download_backtesting_testdata(pair = "BTC-STORJ", interval = 5) + download_backtesting_testdata(pair="BTC-STORJ", interval=5) assert os.path.isfile(file2) is True if os.path.isfile(file2): diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 0890a650e..0d193726d 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -3,7 +3,6 @@ import matplotlib # Install PYQT5 manually if you want to test this helper function matplotlib.use("Qt5Agg") import matplotlib.pyplot as plt - from freqtrade import exchange, analyze @@ -52,4 +51,3 @@ def plot_analyzed_dataframe(pair: str) -> None: if __name__ == '__main__': plot_analyzed_dataframe('BTC_ETH') - From 285308dcbe9f299d6274d4cb76de555db93806f7 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 19 Dec 2017 08:27:52 +0200 Subject: [PATCH 16/64] pass follow redirects for curl to fix travis --- install_ta-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 4b1320d4a..15ab985a3 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,5 +1,5 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then - curl -O http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + curl -O -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib && ./configure && make && sudo make install && cd .. else From 4dab39ed9e4edc589581fd00508bf5a2a4b1abe7 Mon Sep 17 00:00:00 2001 From: seansan Date: Wed, 20 Dec 2017 13:58:18 +0100 Subject: [PATCH 17/64] add % in status table for profit --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f6d6ce234..e63579ee3 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -196,7 +196,7 @@ def _status_table(bot: Bot, update: Update) -> None: trade.id, trade.pair, shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}'.format(100 * trade.calc_profit(current_rate)) + '{:.2f}%'.format(100 * trade.calc_profit(current_rate)) ]) columns = ['ID', 'Pair', 'Since', 'Profit'] From 33beab9c4733dcf43c5134c50a7c4cab454a74ff Mon Sep 17 00:00:00 2001 From: seansan Date: Thu, 21 Dec 2017 09:13:26 +0100 Subject: [PATCH 18/64] added Minimal (advised) system requirements --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f2cf6e8bd..2254296d8 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ if not feel free to raise a github issue. * python3.6 * sqlite * [TA-lib](https://github.com/mrjbq7/ta-lib#dependencies) binaries +* Minimal (advised) system requirements: 2GB RAM, 1GB data, 2vCPU ### Install From 41e22657e4360aaf7003bc31bc18936104bc3126 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Wed, 20 Dec 2017 23:31:26 -0800 Subject: [PATCH 19/64] Fix hyperopt when using MongoDB --- .gitignore | 11 +++-- freqtrade/optimize/__init__.py | 9 ++-- freqtrade/optimize/hyperopt.py | 16 ++------ freqtrade/optimize/hyperopt_conf.py | 41 +++++++++++++++++++ .../tests/test_optimize_hyperopt_config.py | 16 ++++++++ 5 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 freqtrade/optimize/hyperopt_conf.py create mode 100644 freqtrade/tests/test_optimize_hyperopt_config.py diff --git a/.gitignore b/.gitignore index bcf06e9c5..672dd1f6d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ # Freqtrade rules freqtrade/tests/testdata/*.json +hyperopt_conf.py +config.json +*.sqlite +.hyperopt +logfile.txt # Byte-compiled / optimized / DLL files __pycache__/ @@ -76,12 +81,6 @@ target/ # pyenv .python-version -config.json -preprocessor.py -*.sqlite -.hyperopt -logfile.txt - .env .venv .idea diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 12425cda2..e94fd913c 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -5,6 +5,7 @@ import json import os from typing import Optional, List, Dict from freqtrade.exchange import get_ticker_history +from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf from pandas import DataFrame @@ -13,7 +14,7 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe logger = logging.getLogger(__name__) -def load_data(pairs: List[str], ticker_interval: int = 5, +def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None, refresh_pairs: Optional[bool] = False) -> Dict[str, List]: """ Loads ticker history data for the given parameters @@ -24,12 +25,14 @@ def load_data(pairs: List[str], ticker_interval: int = 5, path = testdata_path() result = {} + _pairs = pairs or hyperopt_optimize_conf()['exchange']['pair_whitelist'] + # If the user force the refresh of pairs if refresh_pairs: logger.info('Download data for all pairs and store them in freqtrade/tests/testsdata') - download_pairs(pairs) + download_pairs(_pairs) - for pair in pairs: + for pair in _pairs: file = '{abspath}/{pair}-{ticker_interval}.json'.format( abspath=path, pair=pair, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07e988a91..bf2c85e42 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -16,6 +16,7 @@ from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.misc import load_config from freqtrade.optimize.backtesting import backtest +from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf from freqtrade.vendor.qtpylib.indicators import crossed_above # Remove noisy log messages @@ -35,19 +36,8 @@ AVG_PROFIT_TO_BEAT = 0.2 AVG_DURATION_TO_BEAT = 50 # Configuration and data used by hyperopt -PROCESSED = [] -OPTIMIZE_CONFIG = { - 'max_open_trades': 3, - 'stake_currency': 'BTC', - 'stake_amount': 0.01, - 'minimal_roi': { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, - }, - 'stoploss': -0.10, -} +PROCESSED = optimize.preprocess(optimize.load_data()) +OPTIMIZE_CONFIG = hyperopt_optimize_conf() # Monkey patch config from freqtrade import main # noqa diff --git a/freqtrade/optimize/hyperopt_conf.py b/freqtrade/optimize/hyperopt_conf.py new file mode 100644 index 000000000..cbd95973a --- /dev/null +++ b/freqtrade/optimize/hyperopt_conf.py @@ -0,0 +1,41 @@ +""" +File that contains the configuration for Hyperopt +""" + + +def hyperopt_optimize_conf() -> dict: + """ + This function is used to define which parameters Hyperopt must used. + The "pair_whitelist" is only used is your are using Hyperopt with MongoDB, + without MongoDB, Hyperopt will use the pair your have set in your config file. + :return: + """ + return { + 'max_open_trades': 3, + 'stake_currency': 'BTC', + 'stake_amount': 0.01, + "minimal_roi": { + '40': 0.0, + '30': 0.01, + '20': 0.02, + '0': 0.04, + }, + 'stoploss': -0.10, + "bid_strategy": { + "ask_last_balance": 0.0 + }, + "exchange": { + "pair_whitelist": [ + "BTC_ETH", + "BTC_LTC", + "BTC_ETC", + "BTC_DASH", + "BTC_ZEC", + "BTC_XLM", + "BTC_NXT", + "BTC_POWR", + "BTC_ADA", + "BTC_XMR" + ] + } + } diff --git a/freqtrade/tests/test_optimize_hyperopt_config.py b/freqtrade/tests/test_optimize_hyperopt_config.py new file mode 100644 index 000000000..e06c1f2eb --- /dev/null +++ b/freqtrade/tests/test_optimize_hyperopt_config.py @@ -0,0 +1,16 @@ +# pragma pylint: disable=missing-docstring,W0212 + +from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf + + +def test_hyperopt_optimize_conf(): + hyperopt_conf = hyperopt_optimize_conf() + + assert "max_open_trades" in hyperopt_conf + assert "stake_currency" in hyperopt_conf + assert "stake_amount" in hyperopt_conf + assert "minimal_roi" in hyperopt_conf + assert "stoploss" in hyperopt_conf + assert "bid_strategy" in hyperopt_conf + assert "exchange" in hyperopt_conf + assert "pair_whitelist" in hyperopt_conf['exchange'] From 11585f958185f2645fd22593b88abd7bfb20e817 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Dec 2017 14:29:31 +0200 Subject: [PATCH 20/64] Create contribution guideline --- CONTRIBUTING.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..e8b0684e9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,39 @@ +Feel like our bot is missing a feature? We welcome your pull requests! Few pointers for contributions: + +- Create your PR against the `develop` branch, not `master`. +- New features need to contain unit tests and must be PEP8 conformant (max-line-length = 100). + +If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) +or in a [issue](https://github.com/gcarq/freqtrade/issues) before a PR. + +Before sending the PR: + +## Run unit tests + +All unit tests must pass. If a unit test is broken, change your code to make it pass. It means you have introduced a regression + +**Test the whole project** +```bash +pytest freqtrade +``` + +**Test only one file** +```bash +pytest freqtrade/tests/test_.py +``` + +**Test only one method from one file** +```bash +pytest freqtrade/tests/test_.py:test_ +``` +## Test if your code is PEP8 compliant +**Install packages** (If not already installed) +```bash +pip3.6 install flake8 coveralls +``` +**Run Flake8** +```bash +flake8 freqtrade +``` + + From 95c6ada2add23cc2e4335029b06038bd247cce00 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Dec 2017 14:31:08 +0200 Subject: [PATCH 21/64] link to contribution guide from README.md --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2254296d8..be6bf0975 100644 --- a/README.md +++ b/README.md @@ -254,8 +254,5 @@ $ pytest freqtrade ### Contributing -Feel like our bot is missing a feature? We welcome your pull requests! Few pointers for contributions: - -- Create your PR against the `develop` branch, not `master`. -- New features need to contain unit tests and must be PEP8 conform (`max-line-length = 100`). -- If you are unsure, discuss the feature on [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) or in a [issue](https://github.com/gcarq/freqtrade/issues) before a PR. +We welcome contributions. See our [contribution guide](https://github.com/gcarq/freqtrade/blob/develop/README.md) +for more details. \ No newline at end of file From 8d933636551860839450bcc4b692069680a026e4 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Dec 2017 09:21:04 +0200 Subject: [PATCH 22/64] filter nan values from total_profit and avg_profit --- freqtrade/optimize/hyperopt.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e1d4e3a56..b3dc3d5e4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,6 +11,7 @@ from operator import itemgetter from hyperopt import fmin, tpe, hp, Trials, STATUS_OK from hyperopt.mongoexp import MongoTrials from pandas import DataFrame +import numpy as np from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex @@ -148,7 +149,9 @@ def optimizer(params): return { 'loss': trade_loss + profit_loss, 'status': STATUS_OK, - 'result': result + 'result': result, + 'total_profit': total_profit, + 'avg_profit': result_data['avg_profit'], } @@ -162,6 +165,10 @@ def format_results(results: DataFrame): ) +def filter_nan(result, filter_key): + return [r for r in result if not np.isnan(r[filter_key])] + + def buy_strategy_generator(params): def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] @@ -236,5 +243,10 @@ def start(args): best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials) logger.info('Best parameters:\n%s', json.dumps(best, indent=4)) - results = sorted(trials.results, key=itemgetter('loss')) + + filt_res = filter_nan(trials.results, 'total_profit') + filt_res = filter_nan(filt_res, 'avg_profit') + + results = sorted(filt_res, key=itemgetter('loss')) + logger.info('Best Result:\n%s', results[0]['result']) From 871357a2e35f337cafe1492732d16278e818249d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Dec 2017 16:53:31 +0200 Subject: [PATCH 23/64] just require positive results --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b3dc3d5e4..218e1a910 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -32,9 +32,9 @@ TARGET_TRADES = 1100 TOTAL_TRIES = None _CURRENT_TRIES = 0 -TOTAL_PROFIT_TO_BEAT = 3 -AVG_PROFIT_TO_BEAT = 0.2 -AVG_DURATION_TO_BEAT = 50 +TOTAL_PROFIT_TO_BEAT = 0 +AVG_PROFIT_TO_BEAT = 0 +AVG_DURATION_TO_BEAT = 100 # Configuration and data used by hyperopt PROCESSED = optimize.preprocess(optimize.load_data()) From 10cf2ce853907c431e0eebc8b827e13b22bebc51 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Dec 2017 16:54:04 +0200 Subject: [PATCH 24/64] remove unnecessary confusing division --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 218e1a910..3db11a270 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -101,7 +101,7 @@ def log_results(results): current_try = results['current_tries'] total_tries = results['total_tries'] result = results['result'] - profit = results['total_profit'] / 1000 + profit = results['total_profit'] outcome = '{:5d}/{}: {}'.format(current_try, total_tries, result) From a063680d3204f44e7b597b0553caa08ab2efd84f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Dec 2017 16:55:20 +0200 Subject: [PATCH 25/64] calculate log line only if really logging --- freqtrade/optimize/hyperopt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3db11a270..85c20fe4b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -103,10 +103,8 @@ def log_results(results): result = results['result'] profit = results['total_profit'] - outcome = '{:5d}/{}: {}'.format(current_try, total_tries, result) - if profit >= TOTAL_PROFIT_TO_BEAT: - logger.info(outcome) + logger.info('{:5d}/{}: {}'.format(current_try, total_tries, result)) else: print('.', end='') sys.stdout.flush() From 5309ea3820ad4c708229ca76f34240e55d206dc6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Dec 2017 16:56:59 +0200 Subject: [PATCH 26/64] use newline for each log result for readability --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 85c20fe4b..15af8a649 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -104,7 +104,7 @@ def log_results(results): profit = results['total_profit'] if profit >= TOTAL_PROFIT_TO_BEAT: - logger.info('{:5d}/{}: {}'.format(current_try, total_tries, result)) + logger.info('\n{:5d}/{}: {}'.format(current_try, total_tries, result)) else: print('.', end='') sys.stdout.flush() From 24bc3a8390d7a1c611cc9d7a3e3f3e0855d40894 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Dec 2017 15:08:54 +0200 Subject: [PATCH 27/64] show more digits for profits --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 15af8a649..abf590c4f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -155,7 +155,7 @@ def optimizer(params): def format_results(results: DataFrame): return ('Made {:6d} buys. Average profit {: 5.2f}%. ' - 'Total profit was {: 7.3f}. Average duration {:5.1f} mins.').format( + 'Total profit was {: 11.8f}. Average duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), From 1058820e1bbc3cfba89712768d6b253429db659e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Dec 2017 17:26:22 +0200 Subject: [PATCH 28/64] just pass stake_amount instead of the whole config --- freqtrade/optimize/backtesting.py | 8 ++++---- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/tests/test_optimize_backtesting.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5761c9507..5dc1c3a3d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -64,7 +64,7 @@ def generate_text_table( return tabulate(tabular_data, headers=headers) -def backtest(config: Dict, processed: Dict[str, DataFrame], +def backtest(stake_amount: float, processed: Dict[str, DataFrame], max_open_trades: int = 0, realistic: bool = True) -> DataFrame: """ Implements backtesting functionality @@ -98,8 +98,8 @@ def backtest(config: Dict, processed: Dict[str, DataFrame], trade = Trade( open_rate=row.close, open_date=row.date, - stake_amount=config['stake_amount'], - amount=config['stake_amount'] / row.open, + stake_amount=stake_amount, + amount=stake_amount / row.open, fee=exchange.get_fee() ) @@ -170,7 +170,7 @@ def start(args): # Execute backtest and print results results = backtest( - config, preprocess(data), max_open_trades, args.realistic_simulation + config['stake_amount'], preprocess(data), max_open_trades, args.realistic_simulation ) logger.info( '\n====================== BACKTESTING REPORT ======================================\n%s', diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index abf590c4f..7a3340508 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -116,7 +116,7 @@ def optimizer(params): from freqtrade.optimize import backtesting backtesting.populate_buy_trend = buy_strategy_generator(params) - results = backtest(OPTIMIZE_CONFIG, PROCESSED) + results = backtest(OPTIMIZE_CONFIG['stake_amount'], PROCESSED) result = format_results(results) diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py index f35d21648..2863961d3 100644 --- a/freqtrade/tests/test_optimize_backtesting.py +++ b/freqtrade/tests/test_optimize_backtesting.py @@ -12,7 +12,7 @@ def test_backtest(default_conf, mocker): exchange._API = Bittrex({'key': '', 'secret': ''}) data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) - results = backtest(default_conf, optimize.preprocess(data), 10, True) + results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) num_results = len(results) assert num_results > 0 @@ -23,7 +23,7 @@ def test_1min_ticker_interval(default_conf, mocker): # Run a backtesting for an exiting 5min ticker_interval data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) - results = backtest(default_conf, optimize.preprocess(data), 1, True) + results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 1, True) assert len(results) > 0 From 50e7cef5f39fb506d17da4bc0d9a196beeb7bf9f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Dec 2017 18:01:18 +0200 Subject: [PATCH 29/64] remove commented-out code --- freqtrade/optimize/hyperopt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7a3340508..1e37c1e93 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -140,8 +140,6 @@ def optimizer(params): 'result': result, 'results': results } - - # logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result)) log_results(result_data) return { From e644d57dbe0a493693865725518dad528b381455 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Dec 2017 18:37:42 +0200 Subject: [PATCH 30/64] log should state profit is in BTC to avoid confusion --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 1e37c1e93..4eaaf0f9c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -153,7 +153,7 @@ def optimizer(params): def format_results(results: DataFrame): return ('Made {:6d} buys. Average profit {: 5.2f}%. ' - 'Total profit was {: 11.8f}. Average duration {:5.1f} mins.').format( + 'Total profit was {: 11.8f} BTC. Average duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), From 353b0d2d34f9d80fc6106ea737d9d5e5722c39d7 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Dec 2017 18:38:16 +0200 Subject: [PATCH 31/64] balance hyperopt objective to adjusted profit calculations --- freqtrade/optimize/hyperopt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4eaaf0f9c..2a21c2553 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -36,6 +36,10 @@ TOTAL_PROFIT_TO_BEAT = 0 AVG_PROFIT_TO_BEAT = 0 AVG_DURATION_TO_BEAT = 100 +# this is expexted avg profit * expected trade count +# for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85 +EXPECTED_MAX_PROFIT = 3.85 + # Configuration and data used by hyperopt PROCESSED = optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() @@ -120,11 +124,11 @@ def optimizer(params): result = format_results(results) - total_profit = results.profit_percent.sum() * 1000 + total_profit = results.profit_percent.sum() trade_count = len(results.index) trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) - profit_loss = max(0, 1 - total_profit / 10000) # max profit 10000 + profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) _CURRENT_TRIES += 1 From 67686583007e0a8260d84f525e3a76eef665dcfd Mon Sep 17 00:00:00 2001 From: Pan Long Date: Mon, 25 Dec 2017 14:01:01 +0800 Subject: [PATCH 32/64] Make get_signals async. This should speed up create_trade calls by at least 10x. (#223) --- freqtrade/analyze.py | 3 +++ freqtrade/exchange/__init__.py | 4 +++- freqtrade/main.py | 15 +++++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index d586077db..887fc0282 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -144,6 +144,9 @@ def get_signal(pair: str, signal: SignalType) -> bool: except ValueError as ex: logger.warning('Unable to analyze ticker for pair %s: %s', pair, str(ex)) return False + except Exception: + logger.exception('Unexpected error when analyzing ticker for pair %s.', pair) + return False if dataframe.empty: return False diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 7b5c0c753..8e87cfc8b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -3,6 +3,7 @@ import enum import logging from random import randint +from threading import RLock from typing import List, Dict, Any, Optional import arrow @@ -14,6 +15,7 @@ from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.interface import Exchange logger = logging.getLogger(__name__) +lock = RLock() # Current selected exchange _API: Exchange = None @@ -138,7 +140,7 @@ def get_ticker(pair: str) -> dict: return _API.get_ticker(pair) -@cached(TTLCache(maxsize=100, ttl=30)) +@cached(TTLCache(maxsize=100, ttl=30), lock=lock) def get_ticker_history(pair: str, tick_interval: Optional[int] = 5) -> List[Dict]: return _API.get_ticker_history(pair, tick_interval) diff --git a/freqtrade/main.py b/freqtrade/main.py index 212e9f0e9..31a4c4c55 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 +import asyncio import copy import json import logging import sys import time import traceback +from concurrent.futures import ThreadPoolExecutor from datetime import datetime from typing import Dict, Optional, List @@ -210,8 +212,17 @@ def create_trade(stake_amount: float) -> bool: raise DependencyException('No pair in whitelist') # Pick pair based on StochRSI buy signals - for _pair in whitelist: - if get_signal(_pair, SignalType.BUY): + with ThreadPoolExecutor() as pool: + awaitable_signals = [ + asyncio.wrap_future(pool.submit(get_signal, pair, SignalType.BUY)) + for pair in whitelist + ] + + loop = asyncio.get_event_loop() + signals = loop.run_until_complete(asyncio.gather(*awaitable_signals)) + + for idx, _pair in enumerate(whitelist): + if signals[idx]: pair = _pair break else: From 9959d53f5eaa14b6068985ce081196453c469a93 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Dec 2017 09:18:34 +0200 Subject: [PATCH 33/64] Logging improvements to Hyperopt (#235) * make log texts go on new line * remove unnecessary fields from hyperopt log messages * shorten log text in hyperopt * consider making zero trades a failed hyperopt eval * only log from hyperopt when result improves * remove unnecessary temp variables * remove unused result data variables * remove unused import * fix an outdated comment --- freqtrade/optimize/hyperopt.py | 59 ++++++++++++++-------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2a21c2553..89038a609 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -8,10 +8,9 @@ from functools import reduce from math import exp from operator import itemgetter -from hyperopt import fmin, tpe, hp, Trials, STATUS_OK +from hyperopt import fmin, tpe, hp, Trials, STATUS_OK, STATUS_FAIL from hyperopt.mongoexp import MongoTrials from pandas import DataFrame -import numpy as np from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex @@ -32,9 +31,7 @@ TARGET_TRADES = 1100 TOTAL_TRIES = None _CURRENT_TRIES = 0 -TOTAL_PROFIT_TO_BEAT = 0 -AVG_PROFIT_TO_BEAT = 0 -AVG_DURATION_TO_BEAT = 100 +CURRENT_BEST_LOSS = 100 # this is expexted avg profit * expected trade count # for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85 @@ -100,15 +97,15 @@ SPACE = { def log_results(results): - "if results is better than _TO_BEAT show it" + """ log results if it is better than any previous evaluation """ + global CURRENT_BEST_LOSS - current_try = results['current_tries'] - total_tries = results['total_tries'] - result = results['result'] - profit = results['total_profit'] - - if profit >= TOTAL_PROFIT_TO_BEAT: - logger.info('\n{:5d}/{}: {}'.format(current_try, total_tries, result)) + if results['loss'] < CURRENT_BEST_LOSS: + CURRENT_BEST_LOSS = results['loss'] + logger.info('{:5d}/{}: {}'.format( + results['current_tries'], + results['total_tries'], + results['result'])) else: print('.', end='') sys.stdout.flush() @@ -127,37 +124,37 @@ def optimizer(params): total_profit = results.profit_percent.sum() trade_count = len(results.index) + if trade_count == 0: + return { + 'status': STATUS_FAIL, + 'loss': float('inf') + } + trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - + loss = trade_loss + profit_loss _CURRENT_TRIES += 1 result_data = { - 'trade_count': trade_count, - 'total_profit': total_profit, - 'trade_loss': trade_loss, - 'profit_loss': profit_loss, - 'avg_profit': results.profit_percent.mean() * 100.0, - 'avg_duration': results.duration.mean() * 5, + 'loss': loss, 'current_tries': _CURRENT_TRIES, 'total_tries': TOTAL_TRIES, 'result': result, - 'results': results } log_results(result_data) return { - 'loss': trade_loss + profit_loss, + 'loss': loss, 'status': STATUS_OK, 'result': result, 'total_profit': total_profit, - 'avg_profit': result_data['avg_profit'], + 'avg_profit': results.profit_percent.mean() * 100.0, } def format_results(results: DataFrame): - return ('Made {:6d} buys. Average profit {: 5.2f}%. ' - 'Total profit was {: 11.8f} BTC. Average duration {:5.1f} mins.').format( + return ('{:6d} trades. Avg profit {: 5.2f}%. ' + 'Total profit {: 11.8f} BTC. Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), @@ -165,10 +162,6 @@ def format_results(results: DataFrame): ) -def filter_nan(result, filter_key): - return [r for r in result if not np.isnan(r[filter_key])] - - def buy_strategy_generator(params): def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] @@ -223,7 +216,7 @@ def start(args): # Initialize logger logging.basicConfig( level=args.loglevel, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + format='\n%(message)s', ) logger.info('Using config: %s ...', args.config) @@ -244,9 +237,5 @@ def start(args): best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials) logger.info('Best parameters:\n%s', json.dumps(best, indent=4)) - filt_res = filter_nan(trials.results, 'total_profit') - filt_res = filter_nan(filt_res, 'avg_profit') - - results = sorted(filt_res, key=itemgetter('loss')) - + results = sorted(trials.results, key=itemgetter('loss')) logger.info('Best Result:\n%s', results[0]['result']) From de33d69eed25c305196deee0deb4bf9c80db72e1 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Dec 2017 13:07:50 +0200 Subject: [PATCH 34/64] Lint fixes (#236) * correct docstring * add type annotation to trade_count_lock * fix indentations * allow globals in hyperopt.py * fix import order * simplify asserts * use proper variable name * simplify condition * fix path operation that fails on windows --- freqtrade/main.py | 7 +++---- freqtrade/optimize/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/optimize/hyperopt.py | 4 ++-- freqtrade/tests/test_optimize_backtesting.py | 9 ++++----- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 31a4c4c55..a3a5d49b5 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -78,8 +78,8 @@ def _process(dynamic_whitelist: Optional[int] = 0) -> bool: 'Checked all whitelisted currencies. ' 'Found no suitable entry positions for buying. Will keep looking ...' ) - except DependencyException as e: - logger.warning('Unable to create trade: %s', e) + except DependencyException as exception: + logger.warning('Unable to create trade: %s', exception) for trade in trades: # Get order details for actual price per unit @@ -291,8 +291,7 @@ def gen_pair_whitelist(base_currency: str, topn: int = 20, key: str = 'BaseVolum reverse=True ) - # topn must be greater than 0 - if not topn > 0: + if topn <= 0: topn = 20 return [s['MarketName'].replace('-', '_') for s in summaries[:topn]] diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e94fd913c..a77576a27 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -58,7 +58,7 @@ def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: def testdata_path() -> str: """Return the path where testdata files are stored""" - return os.path.abspath(os.path.dirname(__file__)) + '/../tests/testdata' + return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'tests', 'testdata')) def download_pairs(pairs: List[str]) -> bool: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5dc1c3a3d..2684d2576 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -68,14 +68,14 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], max_open_trades: int = 0, realistic: bool = True) -> DataFrame: """ Implements backtesting functionality - :param config: config to use + :param stake_amount: btc amount to use for each trade :param processed: a processed dictionary with format {pair, data} :param max_open_trades: maximum number of concurrent trades (default: 0, disabled) :param realistic: do we try to simulate realistic trades? (default: True) :return: DataFrame """ trades = [] - trade_count_lock = {} + trade_count_lock: dict = {} exchange._API = Bittrex({'key': '', 'secret': ''}) for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 @@ -120,7 +120,7 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], current_profit_percent, current_profit_BTC, row2.Index - row.Index - ) + ) ) break labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 89038a609..dcdafc946 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring,W0212 +# pragma pylint: disable=missing-docstring,W0212,W0603 import json @@ -159,7 +159,7 @@ def format_results(results: DataFrame): results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), results.duration.mean() * 5, - ) + ) def buy_strategy_generator(params): diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py index 2863961d3..558cfd8d1 100644 --- a/freqtrade/tests/test_optimize_backtesting.py +++ b/freqtrade/tests/test_optimize_backtesting.py @@ -1,10 +1,10 @@ # pragma pylint: disable=missing-docstring,W0212 +import os from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.backtesting import backtest from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata -import os def test_backtest(default_conf, mocker): @@ -13,8 +13,7 @@ def test_backtest(default_conf, mocker): data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) - num_results = len(results) - assert num_results > 0 + assert not results.empty def test_1min_ticker_interval(default_conf, mocker): @@ -24,7 +23,7 @@ def test_1min_ticker_interval(default_conf, mocker): # Run a backtesting for an exiting 5min ticker_interval data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 1, True) - assert len(results) > 0 + assert not results.empty def test_backtest_with_new_pair(default_conf, ticker_history, mocker): @@ -43,7 +42,7 @@ def test_backtest_with_new_pair(default_conf, ticker_history, mocker): def test_testdata_path(): - assert str('freqtrade/optimize/../tests/testdata') in testdata_path() + assert os.path.join('freqtrade', 'tests', 'testdata') in testdata_path() def test_download_pairs(default_conf, ticker_history, mocker): From a514b92dcf72446d417924f0fa4ca9e82a0b1597 Mon Sep 17 00:00:00 2001 From: Michael Egger Date: Tue, 26 Dec 2017 09:39:29 +0100 Subject: [PATCH 35/64] catch MIN_TRADE_REQUIREMENT_NOT_MET as non-critical exception (#237) * add MIN_TRADE_REQUIREMENT_NOT_MET to response validation * implement test --- freqtrade/exchange/bittrex.py | 8 ++++-- freqtrade/tests/test_exchange_bittrex.py | 32 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 freqtrade/tests/test_exchange_bittrex.py diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index ca7fce262..3714de070 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -45,8 +45,12 @@ class Bittrex(Exchange): Validates the given bittrex response and raises a ContentDecodingError if a non-fatal issue happened. """ - if response['message'] == 'NO_API_RESPONSE': - raise ContentDecodingError('Unable to decode bittrex response') + temp_error_messages = [ + 'NO_API_RESPONSE', + 'MIN_TRADE_REQUIREMENT_NOT_MET', + ] + if response['message'] in temp_error_messages: + raise ContentDecodingError('Got {}'.format(response['message'])) @property def fee(self) -> float: diff --git a/freqtrade/tests/test_exchange_bittrex.py b/freqtrade/tests/test_exchange_bittrex.py new file mode 100644 index 000000000..53ca71a83 --- /dev/null +++ b/freqtrade/tests/test_exchange_bittrex.py @@ -0,0 +1,32 @@ +# pragma pylint: disable=missing-docstring,C0103 + +import pytest +from requests.exceptions import ContentDecodingError + +from freqtrade.exchange import Bittrex + + +def test_validate_response_success(): + response = { + 'message': '', + 'result': [], + } + Bittrex._validate_response(response) + + +def test_validate_response_no_api_response(): + response = { + 'message': 'NO_API_RESPONSE', + 'result': None, + } + with pytest.raises(ContentDecodingError, match=r'.*NO_API_RESPONSE.*'): + Bittrex._validate_response(response) + + +def test_validate_response_min_trade_requirement_not_met(): + response = { + 'message': 'MIN_TRADE_REQUIREMENT_NOT_MET', + 'result': None, + } + with pytest.raises(ContentDecodingError, match=r'.*MIN_TRADE_REQUIREMENT_NOT_MET.*'): + Bittrex._validate_response(response) From ff6b0fc1c9266f2177d8d59cab9c00dc2fcd03a0 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 24 Dec 2017 23:51:41 -0800 Subject: [PATCH 36/64] Display profits in fiat --- README.md | 6 ++ config.json.example | 1 + freqtrade/fiat_convert.py | 156 +++++++++++++++++++++++++++ freqtrade/main.py | 31 ++++-- freqtrade/misc.py | 1 + freqtrade/rpc/telegram.py | 73 ++++++++++--- freqtrade/tests/conftest.py | 1 + freqtrade/tests/test_fiat_convert.py | 111 +++++++++++++++++++ freqtrade/tests/test_main.py | 3 + freqtrade/tests/test_rpc_telegram.py | 42 ++++++-- requirements.txt | 1 + setup.py | 1 + 12 files changed, 400 insertions(+), 27 deletions(-) create mode 100644 freqtrade/fiat_convert.py create mode 100644 freqtrade/tests/test_fiat_convert.py diff --git a/README.md b/README.md index be6bf0975..286591064 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,12 @@ use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. +`fiat_currency` set the fiat to use for the conversion form coin to +fiat in Telegram. The valid value are: "AUD", "BRL", "CAD", "CHF", +"CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", +"INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", +"RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". + The other values should be self-explanatory, if not feel free to raise a github issue. diff --git a/config.json.example b/config.json.example index cae64aab5..f94e423eb 100644 --- a/config.json.example +++ b/config.json.example @@ -2,6 +2,7 @@ "max_open_trades": 3, "stake_currency": "BTC", "stake_amount": 0.05, + "fiat_display_currency": "USD", "dry_run": false, "minimal_roi": { "40": 0.0, diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py new file mode 100644 index 000000000..567835eed --- /dev/null +++ b/freqtrade/fiat_convert.py @@ -0,0 +1,156 @@ +import logging +import time +from pymarketcap import Pymarketcap + +logger = logging.getLogger(__name__) + + +class CryptoFiat(): + # Constants + CACHE_DURATION = 6 * 60 * 60 # 6 hours + + def __init__(self, crypto_symbol: str, fiat_symbol: str, price: float) -> None: + """ + Create an object that will contains the price for a crypto-currency in fiat + :param crypto_symbol: Crypto-currency you want to convert (e.g BTC) + :param fiat_symbol: FIAT currency you want to convert to (e.g USD) + :param price: Price in FIAT + """ + + # Public attributes + self.crypto_symbol = None + self.fiat_symbol = None + self.price = 0.0 + + # Private attributes + self._expiration = 0 + + self.crypto_symbol = crypto_symbol.upper() + self.fiat_symbol = fiat_symbol.upper() + self.set_price(price=price) + + def set_price(self, price: float) -> None: + """ + Set the price of the Crypto-currency in FIAT and set the expiration time + :param price: Price of the current Crypto currency in the fiat + :return: None + """ + self.price = price + self._expiration = time.time() + self.CACHE_DURATION + + def is_expired(self) -> bool: + """ + Return if the current price is still valid or needs to be refreshed + :return: bool, true the price is expired and needs to be refreshed, false the price is + still valid + """ + return self._expiration - time.time() <= 0 + + +class CryptoToFiatConverter(): + # Constants + SUPPORTED_FIAT = [ + "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", + "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", + "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", + "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" + ] + + def __init__(self) -> None: + self._coinmarketcap = Pymarketcap() + self._pairs = [] + + def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float: + """ + Convert an amount of crypto-currency to fiat + :param crypto_amount: amount of crypto-currency to convert + :param crypto_symbol: crypto-currency used + :param fiat_symbol: fiat to convert to + :return: float, value in fiat of the crypto-currency amount + """ + price = self.get_price(crypto_symbol=crypto_symbol, fiat_symbol=fiat_symbol) + return float(crypto_amount) * float(price) + + def get_price(self, crypto_symbol: str, fiat_symbol: str) -> float: + """ + Return the price of the Crypto-currency in Fiat + :param crypto_symbol: Crypto-currency you want to convert (e.g BTC) + :param fiat_symbol: FIAT currency you want to convert to (e.g USD) + :return: Price in FIAT + """ + crypto_symbol = crypto_symbol.upper() + fiat_symbol = fiat_symbol.upper() + + # Check if the fiat convertion you want is supported + if not self._is_supported_fiat(fiat=fiat_symbol): + raise ValueError('The fiat {} is not supported.'.format(fiat_symbol)) + + # Get the pair that interest us and return the price in fiat + for pair in self._pairs: + if pair.crypto_symbol == crypto_symbol and pair.fiat_symbol == fiat_symbol: + # If the price is expired we refresh it, avoid to call the API all the time + if pair.is_expired(): + pair.set_price( + price=self._find_price( + crypto_symbol=pair.crypto_symbol, + fiat_symbol=pair.fiat_symbol + ) + ) + + # return the last price we have for this pair + return pair.price + + # The pair does not exist, so we create it and return the price + return self._add_pair( + crypto_symbol=crypto_symbol, + fiat_symbol=fiat_symbol, + price=self._find_price( + crypto_symbol=crypto_symbol, + fiat_symbol=fiat_symbol + ) + ) + + def _add_pair(self, crypto_symbol: str, fiat_symbol: str, price: float) -> float: + """ + :param crypto_symbol: Crypto-currency you want to convert (e.g BTC) + :param fiat_symbol: FIAT currency you want to convert to (e.g USD) + :return: price in FIAT + """ + self._pairs.append( + CryptoFiat( + crypto_symbol=crypto_symbol, + fiat_symbol=fiat_symbol, + price=price + ) + ) + + return price + + def _is_supported_fiat(self, fiat: str) -> bool: + """ + Check if the FIAT your want to convert to is supported + :param fiat: FIAT to check (e.g USD) + :return: bool, True supported, False not supported + """ + + fiat = fiat.upper() + + return fiat in self.SUPPORTED_FIAT + + def _find_price(self, crypto_symbol: str, fiat_symbol: str) -> float: + """ + Call CoinMarketCap API to retrieve the price in the FIAT + :param crypto_symbol: Crypto-currency you want to convert (e.g BTC) + :param fiat_symbol: FIAT currency you want to convert to (e.g USD) + :return: float, price of the crypto-currency in Fiat + """ + # Check if the fiat convertion you want is supported + if not self._is_supported_fiat(fiat=fiat_symbol): + raise ValueError('The fiat {} is not supported.'.format(fiat_symbol)) + + return float( + self._coinmarketcap.ticker( + currency=crypto_symbol, + convert=fiat_symbol + )['price_' + fiat_symbol.lower()] + ) diff --git a/freqtrade/main.py b/freqtrade/main.py index 212e9f0e9..47ff599b2 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -17,6 +17,7 @@ from freqtrade.analyze import get_signal, SignalType from freqtrade.misc import State, get_state, update_state, parse_args, throttle, \ load_config from freqtrade.persistence import Trade +from freqtrade.fiat_convert import CryptoToFiatConverter logger = logging.getLogger('freqtrade') @@ -119,14 +120,28 @@ def execute_sell(trade: Trade, limit: float) -> None: trade.open_order_id = order_id fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) - rpc.send_msg('*{}:* Selling [{}]({}) with limit `{:.8f} (profit: ~{:.2f}%, {:.8f})`'.format( - trade.exchange, - trade.pair.replace('_', '/'), - exchange.get_pair_detail_url(trade.pair), - limit, - fmt_exp_profit, - trade.calc_profit(rate=limit), - )) + profit_trade = trade.calc_profit(rate=limit) + + fiat_converter = CryptoToFiatConverter() + profit_fiat = fiat_converter.convert_amount( + profit_trade, + _CONF['stake_currency'], + _CONF['fiat_display_currency'] + ) + + rpc.send_msg('*{exchange}:* Selling [{pair}]({pair_url}) with limit `{limit:.8f}`' + '` (profit: ~{profit_percent:.2f}%, {profit_coin:.8f} {coin}`' + '` / {profit_fiat:.3f} {fiat})`'.format( + exchange=trade.exchange, + pair=trade.pair.replace('_', '/'), + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit, + profit_percent=fmt_exp_profit, + profit_coin=profit_trade, + coin=_CONF['stake_currency'], + profit_fiat=profit_fiat, + fiat=_CONF['fiat_display_currency'], + )) Trade.session.flush() diff --git a/freqtrade/misc.py b/freqtrade/misc.py index b01fd9fe9..57e7c6735 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -208,6 +208,7 @@ CONF_SCHEMA = { 'max_open_trades': {'type': 'integer', 'minimum': 1}, 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, 'stake_amount': {'type': 'number', 'minimum': 0.0005}, + 'fiat_display_currency': {'type': 'string', 'enum': ['USD', 'EUR', 'CAD', 'SGD']}, 'dry_run': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d62d491e1..79290d159 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -15,6 +15,7 @@ from telegram.ext import CommandHandler, Updater from freqtrade import exchange, __version__ from freqtrade.misc import get_state, State, update_state from freqtrade.persistence import Trade +from freqtrade.fiat_convert import CryptoToFiatConverter # Remove noisy log messages logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) @@ -23,6 +24,7 @@ logger = logging.getLogger(__name__) _UPDATER: Updater = None _CONF = {} +_FIAT_CONVERT = CryptoToFiatConverter() def init(config: dict) -> None: @@ -242,8 +244,28 @@ def _daily(bot: Bot, update: Update) -> None: curdayprofit = sum(trade.calc_profit() for trade in trades) profit_days[date.fromordinal(today - day)] = format(curdayprofit, '.8f') - stats = [[key, str(value) + ' BTC'] for key, value in profit_days.items()] - stats = tabulate(stats, headers=['Day', 'Profit'], tablefmt='simple') + stats = [ + [ + key, + '{value:.8f} {symbol}'.format(value=float(value), symbol=_CONF['stake_currency']), + '{value:.3f} {symbol}'.format( + value=_FIAT_CONVERT.convert_amount( + value, + _CONF['stake_currency'], + _CONF['fiat_display_currency'] + ), + symbol=_CONF['fiat_display_currency'] + ) + ] + for key, value in profit_days.items() + ] + stats = tabulate(stats, + headers=[ + 'Day', + 'Profit {}'.format(_CONF['stake_currency']), + 'Profit {}'.format(_CONF['fiat_display_currency']) + ], + tablefmt='simple') message = 'Daily Profit over the last {} days:\n
{}
'.format(timescale, stats) send_msg(message, bot=bot, parse_mode=ParseMode.HTML) @@ -260,9 +282,9 @@ def _profit(bot: Bot, update: Update) -> None: """ trades = Trade.query.order_by(Trade.id).all() - profit_all_btc = [] + profit_all_coin = [] profit_all_percent = [] - profit_btc_closed = [] + profit_closed_coin = [] profit_closed_percent = [] durations = [] @@ -276,14 +298,14 @@ def _profit(bot: Bot, update: Update) -> None: if not trade.is_open: profit_percent = trade.calc_profit_percent() - profit_btc_closed.append(trade.calc_profit()) + profit_closed_coin.append(trade.calc_profit()) profit_closed_percent.append(profit_percent) else: # Get current rate current_rate = exchange.get_ticker(trade.pair)['bid'] profit_percent = trade.calc_profit_percent(rate=current_rate) - profit_all_btc.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate))) + profit_all_coin.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate))) profit_all_percent.append(profit_percent) best_pair = Trade.session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ @@ -297,19 +319,46 @@ def _profit(bot: Bot, update: Update) -> None: return bp_pair, bp_rate = best_pair + + # Prepare data to display + profit_closed_coin = round(sum(profit_closed_coin), 8) + profit_closed_percent = round(sum(profit_closed_percent) * 100, 2) + profit_closed_fiat = _FIAT_CONVERT.convert_amount( + profit_closed_coin, + _CONF['stake_currency'], + _CONF['fiat_display_currency'] + ) + profit_all_coin = round(sum(profit_all_coin), 8) + profit_all_percent = round(sum(profit_all_percent) * 100, 2) + profit_all_fiat = _FIAT_CONVERT.convert_amount( + profit_all_coin, + _CONF['stake_currency'], + _CONF['fiat_display_currency'] + ) + + # Message to display markdown_msg = """ -*ROI Trade closed:* `{profit_closed_btc:.8f} BTC ({profit_closed_percent:.2f}%)` -*ROI All trades:* `{profit_all_btc:.8f} BTC ({profit_all_percent:.2f}%)` +*ROI:* Close trades + ∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)` + ∙ `{profit_closed_fiat:.3f} {fiat}` +*ROI:* All trades + ∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)` + ∙ `{profit_all_fiat:.3f} {fiat}` + *Total Trade Count:* `{trade_count}` *First Trade opened:* `{first_trade_date}` *Latest Trade opened:* `{latest_trade_date}` *Avg. Duration:* `{avg_duration}` *Best Performing:* `{best_pair}: {best_rate:.2f}%` """.format( - profit_closed_btc=round(sum(profit_btc_closed), 8), - profit_closed_percent=round(sum(profit_closed_percent) * 100, 2), - profit_all_btc=round(sum(profit_all_btc), 8), - profit_all_percent=round(sum(profit_all_percent) * 100, 2), + coin=_CONF['stake_currency'], + fiat=_CONF['fiat_display_currency'], + profit_closed_coin=profit_closed_coin, + profit_closed_percent=profit_closed_percent, + profit_closed_fiat=profit_closed_fiat, + profit_all_coin=profit_all_coin, + profit_all_percent=profit_all_percent, + profit_all_fiat=profit_all_fiat, trade_count=len(trades), first_trade_date=arrow.get(trades[0].open_date).humanize(), latest_trade_date=arrow.get(trades[-1].open_date).humanize(), diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index b034b8c9f..c8ecd39c7 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -16,6 +16,7 @@ def default_conf(): "max_open_trades": 1, "stake_currency": "BTC", "stake_amount": 0.001, + "fiat_display_currency": "USD", "dry_run": True, "minimal_roi": { "40": 0.0, diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py new file mode 100644 index 000000000..13597f3a3 --- /dev/null +++ b/freqtrade/tests/test_fiat_convert.py @@ -0,0 +1,111 @@ +# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 + +import time +import pytest +from unittest.mock import MagicMock + +from freqtrade.fiat_convert import CryptoToFiatConverter, CryptoFiat + + +def test_pair_convertion_object(): + pair_convertion = CryptoFiat( + crypto_symbol='btc', + fiat_symbol='usd', + price=12345.0 + ) + + # Check the cache duration is 6 hours + assert pair_convertion.CACHE_DURATION == 6 * 60 * 60 + + # Check a regular usage + assert pair_convertion.crypto_symbol == 'BTC' + assert pair_convertion.fiat_symbol == 'USD' + assert pair_convertion.price == 12345.0 + assert pair_convertion.is_expired() is False + + # Update the expiration time (- 2 hours) and check the behavior + pair_convertion._expiration = time.time() - 2 * 60 * 60 + assert pair_convertion.is_expired() is True + + # Check set price behaviour + time_reference = time.time() + pair_convertion.CACHE_DURATION + pair_convertion.set_price(price=30000.123) + assert pair_convertion.is_expired() is False + assert pair_convertion._expiration >= time_reference + assert pair_convertion.price == 30000.123 + + +def test_fiat_convert_is_supported(): + fiat_convert = CryptoToFiatConverter() + assert fiat_convert._is_supported_fiat(fiat='USD') is True + assert fiat_convert._is_supported_fiat(fiat='usd') is True + assert fiat_convert._is_supported_fiat(fiat='abc') is False + assert fiat_convert._is_supported_fiat(fiat='ABC') is False + + +def test_fiat_convert_add_pair(): + fiat_convert = CryptoToFiatConverter() + + assert len(fiat_convert._pairs) == 0 + + fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='usd', price=12345.0) + assert len(fiat_convert._pairs) == 1 + assert fiat_convert._pairs[0].crypto_symbol == 'BTC' + assert fiat_convert._pairs[0].fiat_symbol == 'USD' + assert fiat_convert._pairs[0].price == 12345.0 + + fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='Eur', price=13000.2) + assert len(fiat_convert._pairs) == 2 + assert fiat_convert._pairs[1].crypto_symbol == 'BTC' + assert fiat_convert._pairs[1].fiat_symbol == 'EUR' + assert fiat_convert._pairs[1].price == 13000.2 + + +def test_fiat_convert_find_price(mocker): + api_mock = MagicMock(return_value={ + 'price_usd': 12345.0, + 'price_eur': 13000.2 + }) + mocker.patch('freqtrade.fiat_convert.Pymarketcap.ticker', api_mock) + fiat_convert = CryptoToFiatConverter() + + with pytest.raises(ValueError, match=r'The fiat ABC is not supported.'): + fiat_convert._find_price(crypto_symbol='BTC', fiat_symbol='ABC') + + assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 12345.0 + assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 12345.0 + assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='EUR') == 13000.2 + + +def test_fiat_convert_get_price(mocker): + api_mock = MagicMock(return_value={ + 'price_usd': 28000.0, + 'price_eur': 15000.0 + }) + mocker.patch('freqtrade.fiat_convert.Pymarketcap.ticker', api_mock) + + fiat_convert = CryptoToFiatConverter() + + with pytest.raises(ValueError, match=r'The fiat US DOLLAR is not supported.'): + fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='US Dollar') + + # Check the value return by the method + assert len(fiat_convert._pairs) == 0 + assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 28000.0 + assert fiat_convert._pairs[0].crypto_symbol == 'BTC' + assert fiat_convert._pairs[0].fiat_symbol == 'USD' + assert fiat_convert._pairs[0].price == 28000.0 + assert fiat_convert._pairs[0]._expiration is not 0 + assert len(fiat_convert._pairs) == 1 + + # Verify the cached is used + fiat_convert._pairs[0].price = 9867.543 + expiration = fiat_convert._pairs[0]._expiration + assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 9867.543 + assert fiat_convert._pairs[0]._expiration == expiration + + # Verify the cache expiration + expiration = time.time() - 2 * 60 * 60 + fiat_convert._pairs[0]._expiration = expiration + assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 28000.0 + assert fiat_convert._pairs[0]._expiration is not expiration diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index d9b774a9d..7c06936c0 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -192,6 +192,9 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): }), buy=MagicMock(return_value='mocked_limit_buy'), sell=MagicMock(return_value='mocked_limit_sell')) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) init(default_conf, create_engine('sqlite://')) create_trade(0.001) diff --git a/freqtrade/tests/test_rpc_telegram.py b/freqtrade/tests/test_rpc_telegram.py index 3b6197f23..5a5765ba2 100644 --- a/freqtrade/tests/test_rpc_telegram.py +++ b/freqtrade/tests/test_rpc_telegram.py @@ -164,6 +164,9 @@ def test_profit_handle( mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) init(default_conf, create_engine('sqlite://')) _profit(bot=MagicMock(), update=update) @@ -194,9 +197,14 @@ def test_profit_handle( _profit(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert '*ROI Trade closed:* `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] - assert '*ROI All trades:* `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] - assert 'Best Performing:* `BTC_ETH: 6.20%`' in msg_mock.call_args_list[-1][0][0] + assert '*ROI:* Close trades' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + + assert '*Best Performing:* `BTC_ETH: 6.20%`' in msg_mock.call_args_list[-1][0][0] def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker): @@ -210,6 +218,9 @@ def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker): mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) init(default_conf, create_engine('sqlite://')) # Create some test data @@ -228,7 +239,9 @@ def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker): assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172 (profit: ~6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: ~6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, mocker): @@ -242,6 +255,9 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) init(default_conf, create_engine('sqlite://')) # Create some test data @@ -260,7 +276,9 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044 (profit: ~-5.48%, -0.00005492)' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: ~-5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] def test_exec_forcesell_open_orders(default_conf, ticker, mocker): @@ -298,6 +316,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, mocker): mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) init(default_conf, create_engine('sqlite://')) # Create some test data @@ -310,7 +331,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, mocker): assert rpc_mock.call_count == 4 for args in rpc_mock.call_args_list: - assert '0.00001098 (profit: ~-0.59%, -0.00000591)' in args[0][0] + assert '0.00001098' in args[0][0] + assert 'profit: ~-0.59%, -0.00000591 BTC' in args[0][0] + assert '-0.089 USD' in args[0][0] def test_forcesell_handle_invalid(default_conf, update, mocker): @@ -397,6 +420,9 @@ def test_daily_handle( mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) init(default_conf, create_engine('sqlite://')) # Create some test data @@ -418,7 +444,9 @@ def test_daily_handle( _daily(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'Daily' in msg_mock.call_args_list[0][0][0] - assert str(datetime.utcnow().date()) + ' 0.00006217 BTC' in msg_mock.call_args_list[0][0][0] + assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] + assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] + assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] # Try invalid data msg_mock.reset_mock() diff --git a/requirements.txt b/requirements.txt index 712533cb5..b37a15c79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 +pymarketcap==3.3.139 # Required for plotting data #matplotlib==2.1.0 diff --git a/setup.py b/setup.py index 1514f6405..e53606dea 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ setup(name='freqtrade', 'TA-Lib', 'tabulate', 'cachetools', + 'pymarketcap', ], include_package_data=True, zip_safe=False, From 646437363676156533ba00bb4dd6079b8ebc52d0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 27 Dec 2017 10:19:45 +0100 Subject: [PATCH 37/64] Update pymarketcap from 3.3.139 to 3.3.141 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b37a15c79..e36b66dc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.139 +pymarketcap==3.3.141 # Required for plotting data #matplotlib==2.1.0 From 6c8253a4f57571d210cca60c41da86c5d03cafe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rald=20LONLAS?= Date: Wed, 27 Dec 2017 02:41:11 -0800 Subject: [PATCH 38/64] Add more unittest (#241) --- freqtrade/persistence.py | 2 +- freqtrade/tests/test_main.py | 25 ++++++++ freqtrade/tests/test_persistence.py | 98 ++++++++++++++++++++++++++++- 3 files changed, 123 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 9825299b7..d50c9acb4 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -104,7 +104,7 @@ class Trade(_DECL_BASE): self.close(order['rate']) else: raise ValueError('Unknown order type: {}'.format(order['type'])) - Trade.session.flush() + cleanup() def close(self, rate: float) -> None: """ diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 7c06936c0..037d6f836 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock import pytest import requests +import logging from sqlalchemy import create_engine from freqtrade import DependencyException, OperationalException @@ -216,6 +217,30 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.close_date is not None +def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): + default_conf.update({'experimental': {'use_sell_signal': True}}) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch('freqtrade.main.min_roi_reached', return_value=True) + + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) + + trade = Trade.query.first() + trade.is_open = True + + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: False) + value_returned = handle_trade(trade) + assert ('freqtrade', logging.DEBUG, 'Checking sell_signal ...') in caplog.record_tuples + assert value_returned is False + + def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index f91a8efd0..c2e2c13ea 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,8 +1,91 @@ # pragma pylint: disable=missing-docstring import pytest +import os from freqtrade.exchange import Exchanges -from freqtrade.persistence import Trade +from freqtrade.persistence import init, Trade + + +def test_init_create_session(default_conf, mocker): + mocker.patch.dict('freqtrade.persistence._CONF', default_conf) + + # Check if init create a session + init(default_conf) + assert hasattr(Trade, 'session') + assert type(Trade.session).__name__ is 'Session' + + +def test_init_dry_run_db(default_conf, mocker): + default_conf.update({'dry_run_db': True}) + mocker.patch.dict('freqtrade.persistence._CONF', default_conf) + + # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data) + dry_run_db = 'tradesv3.dry_run.sqlite' + dry_run_db_swp = dry_run_db + '.swp' + + if os.path.isfile(dry_run_db): + os.rename(dry_run_db, dry_run_db_swp) + + # Check if the new tradesv3.dry_run.sqlite was created + init(default_conf) + assert os.path.isfile(dry_run_db) is True + + # Delete the file made for this unitest and rollback to the previous + # tradesv3.dry_run.sqlite file + + # 1. Delete file from the test + if os.path.isfile(dry_run_db): + os.remove(dry_run_db) + + # 2. Rollback to the initial file + if os.path.isfile(dry_run_db_swp): + os.rename(dry_run_db_swp, dry_run_db) + + +def test_init_dry_run_without_db(default_conf, mocker): + default_conf.update({'dry_run_db': False}) + mocker.patch.dict('freqtrade.persistence._CONF', default_conf) + + # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data) + dry_run_db = 'tradesv3.dry_run.sqlite' + dry_run_db_swp = dry_run_db + '.swp' + + if os.path.isfile(dry_run_db): + os.rename(dry_run_db, dry_run_db_swp) + + # Check if the new tradesv3.dry_run.sqlite was created + init(default_conf) + assert os.path.isfile(dry_run_db) is False + + # Rollback to the initial 'tradesv3.dry_run.sqlite' file + if os.path.isfile(dry_run_db_swp): + os.rename(dry_run_db_swp, dry_run_db) + + +def test_init_prod_db(default_conf, mocker): + default_conf.update({'dry_run': False}) + mocker.patch.dict('freqtrade.persistence._CONF', default_conf) + + # First, protect the existing 'tradesv3.sqlite' (Do not delete user data) + prod_db = 'tradesv3.sqlite' + prod_db_swp = prod_db + '.swp' + + if os.path.isfile(prod_db): + os.rename(prod_db, prod_db_swp) + + # Check if the new tradesv3.sqlite was created + init(default_conf) + assert os.path.isfile(prod_db) is True + + # Delete the file made for this unitest and rollback to the previous tradesv3.sqlite file + + # 1. Delete file from the test + if os.path.isfile(prod_db): + os.remove(prod_db) + + # Rollback to the initial 'tradesv3.sqlite' file + if os.path.isfile(prod_db_swp): + os.rename(prod_db_swp, prod_db) def test_update_with_bittrex(limit_buy_order, limit_sell_order): @@ -81,6 +164,19 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order): assert trade.calc_profit_percent() == 0.06201057 +def test_calc_close_trade_price_exception(limit_buy_order): + trade = Trade( + pair='BTC_ETH', + stake_amount=0.001, + fee=0.0025, + exchange=Exchanges.BITTREX, + ) + + trade.open_order_id = 'something' + trade.update(limit_buy_order) + assert trade.calc_close_trade_price() == 0.0 + + def test_update_open_order(limit_buy_order): trade = Trade( pair='BTC_ETH', From 9b4c0f01f2b19ad556f337d290e1823e8f0014f7 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 08:05:49 +0200 Subject: [PATCH 39/64] more unit tests for backtesting --- freqtrade/tests/test_optimize_backtesting.py | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py index 558cfd8d1..99f95d9bf 100644 --- a/freqtrade/tests/test_optimize_backtesting.py +++ b/freqtrade/tests/test_optimize_backtesting.py @@ -1,12 +1,36 @@ # pragma pylint: disable=missing-docstring,W0212 import os +import pandas as pd from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex -from freqtrade.optimize.backtesting import backtest +from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata +def test_generate_text_table(): + results = pd.DataFrame( + { + 'currency': ['BTC_ETH', 'BTC_ETH'], + 'profit_percent': [0.1, 0.2], + 'profit_BTC': [0.2, 0.4], + 'duration': [10, 30] + } + ) + assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == ( + 'pair buy count avg profit total profit avg duration\n' + '------- ----------- ------------ -------------- --------------\n' + 'BTC_ETH 2 15.00% 0.60000000 BTC 100\n' + 'TOTAL 2 15.00% 0.60000000 BTC 100') + + +def test_get_timeframe(): + data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) + min_date, max_date = get_timeframe(data) + assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' + assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' + + def test_backtest(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) exchange._API = Bittrex({'key': '', 'secret': ''}) From d61d88559cae01b42b4aacf8c99ce56743efed9d Mon Sep 17 00:00:00 2001 From: Jean Baptiste LE STANG Date: Wed, 27 Dec 2017 21:06:05 +0100 Subject: [PATCH 40/64] Fixing daily profit, taking into account the time part of the date (removing it in fact) --- freqtrade/rpc/telegram.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 79290d159..4661459b1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -220,29 +220,28 @@ def _daily(bot: Bot, update: Update) -> None: :param update: message update :return: None """ - today = datetime.utcnow().toordinal() + today = datetime.utcnow().date() profit_days = {} try: timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): - timescale = 5 + timescale = 7 if not (isinstance(timescale, int) and timescale > 0): send_msg('*Daily [n]:* `must be an integer greater than 0`', bot=bot) return for day in range(0, timescale): - # need to query between day+1 and day-1 - nextdate = date.fromordinal(today - day + 1) - prevdate = date.fromordinal(today - day - 1) + profitday = today - timedelta(days=day) trades = Trade.query \ .filter(Trade.is_open.is_(False)) \ - .filter(between(Trade.close_date, prevdate, nextdate)) \ + .filter(Trade.close_date >= profitday)\ + .filter(Trade.close_date < (profitday + timedelta(days=1)))\ .order_by(Trade.close_date)\ .all() curdayprofit = sum(trade.calc_profit() for trade in trades) - profit_days[date.fromordinal(today - day)] = format(curdayprofit, '.8f') + profit_days[profitday] = format(curdayprofit, '.8f') stats = [ [ From 8537e9f40f0bea3e01221e385b5da97a22de2280 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Wed, 27 Dec 2017 21:33:42 +0100 Subject: [PATCH 41/64] CI flake8 error --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4661459b1..7636c2b8a 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,12 +1,12 @@ import logging import re from decimal import Decimal -from datetime import timedelta, date, datetime +from datetime import timedelta, datetime from typing import Callable, Any import arrow from pandas import DataFrame -from sqlalchemy import and_, func, text, between +from sqlalchemy import and_, func, text from tabulate import tabulate from telegram import ParseMode, Bot, Update, ReplyKeyboardMarkup from telegram.error import NetworkError, TelegramError From ae0a1436e2a7bbcd5903e558f0dec87ba398b161 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 18:54:14 +0200 Subject: [PATCH 42/64] match test files to prod files for backtesting/hyperopt --- .../test_backtesting.py} | 0 .../{test_optimize_hyperopt.py => optimize/test_hyperopt.py} | 0 .../test_hyperopt_config.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename freqtrade/tests/{test_optimize_backtesting.py => optimize/test_backtesting.py} (100%) rename freqtrade/tests/{test_optimize_hyperopt.py => optimize/test_hyperopt.py} (100%) rename freqtrade/tests/{test_optimize_hyperopt_config.py => optimize/test_hyperopt_config.py} (100%) diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py similarity index 100% rename from freqtrade/tests/test_optimize_backtesting.py rename to freqtrade/tests/optimize/test_backtesting.py diff --git a/freqtrade/tests/test_optimize_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py similarity index 100% rename from freqtrade/tests/test_optimize_hyperopt.py rename to freqtrade/tests/optimize/test_hyperopt.py diff --git a/freqtrade/tests/test_optimize_hyperopt_config.py b/freqtrade/tests/optimize/test_hyperopt_config.py similarity index 100% rename from freqtrade/tests/test_optimize_hyperopt_config.py rename to freqtrade/tests/optimize/test_hyperopt_config.py From 7b0beb0afa0de4beb545452431499c77162db5f9 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 09:59:38 +0200 Subject: [PATCH 43/64] cleanups --- freqtrade/optimize/__init__.py | 10 +++------- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/optimize/hyperopt.py | 14 +++++--------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index a77576a27..ac077506a 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -4,11 +4,9 @@ import logging import json import os from typing import Optional, List, Dict +from pandas import DataFrame from freqtrade.exchange import get_ticker_history from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf - -from pandas import DataFrame - from freqtrade.analyze import populate_indicators, parse_ticker_dataframe logger = logging.getLogger(__name__) @@ -50,10 +48,8 @@ def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None, def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: """Creates a dataframe and populates indicators for given ticker data""" - processed = {} - for pair, pair_data in tickerdata.items(): - processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data)) - return processed + return {pair: populate_indicators(parse_ticker_dataframe(pair_data)) + for pair, pair_data in tickerdata.items()} def testdata_path() -> str: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2684d2576..984ca3e72 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -111,14 +111,14 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1: current_profit_percent = trade.calc_profit_percent(rate=row2.close) - current_profit_BTC = trade.calc_profit(rate=row2.close) + current_profit_btc = trade.calc_profit(rate=row2.close) lock_pair_until = row2.Index trades.append( ( pair, current_profit_percent, - current_profit_BTC, + current_profit_btc, row2.Index - row.Index ) ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index dcdafc946..686f15a17 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -118,8 +118,7 @@ def optimizer(params): backtesting.populate_buy_trend = buy_strategy_generator(params) results = backtest(OPTIMIZE_CONFIG['stake_amount'], PROCESSED) - - result = format_results(results) + result_explanation = format_results(results) total_profit = results.profit_percent.sum() trade_count = len(results.index) @@ -135,20 +134,17 @@ def optimizer(params): loss = trade_loss + profit_loss _CURRENT_TRIES += 1 - result_data = { + log_results({ 'loss': loss, 'current_tries': _CURRENT_TRIES, 'total_tries': TOTAL_TRIES, - 'result': result, - } - log_results(result_data) + 'result': result_explanation, + }) return { 'loss': loss, 'status': STATUS_OK, - 'result': result, - 'total_profit': total_profit, - 'avg_profit': results.profit_percent.mean() * 100.0, + 'result': result_explanation, } From 7f44ba6df48579659d379d2f7614d67fb91f221d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 10:08:10 +0200 Subject: [PATCH 44/64] unit tests for optimize.hyperopt --- freqtrade/optimize/hyperopt.py | 14 ++-- freqtrade/tests/optimize/test_hyperopt.py | 81 +++++++++++++++++++++-- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 686f15a17..aeb7c1456 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -25,12 +25,10 @@ logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) logger = logging.getLogger(__name__) - # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 TOTAL_TRIES = None _CURRENT_TRIES = 0 - CURRENT_BEST_LOSS = 100 # this is expexted avg profit * expected trade count @@ -111,6 +109,13 @@ def log_results(results): sys.stdout.flush() +def calculate_loss(total_profit: float, trade_count: int): + """ objective function, returns smaller number for more optimal results """ + trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) + profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) + return trade_loss + profit_loss + + def optimizer(params): global _CURRENT_TRIES @@ -129,9 +134,8 @@ def optimizer(params): 'loss': float('inf') } - trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) - profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - loss = trade_loss + profit_loss + loss = calculate_loss(total_profit, trade_count) + _CURRENT_TRIES += 1 log_results({ diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a8bfe7dd4..104b3bfdd 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,6 +1,79 @@ -# pragma pylint: disable=missing-docstring,W0212 +# pragma pylint: disable=missing-docstring,W0212,C0103 + +from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ + log_results -def test_optimizer(default_conf, mocker): - # TODO: implement test - pass +def test_loss_calculation_prefer_correct_trade_count(): + correct = calculate_loss(1, TARGET_TRADES) + over = calculate_loss(1, TARGET_TRADES + 100) + under = calculate_loss(1, TARGET_TRADES - 100) + assert over > correct + assert under > correct + + +def test_loss_calculation_has_limited_profit(): + correct = calculate_loss(EXPECTED_MAX_PROFIT, TARGET_TRADES) + over = calculate_loss(EXPECTED_MAX_PROFIT * 2, TARGET_TRADES) + under = calculate_loss(EXPECTED_MAX_PROFIT / 2, TARGET_TRADES) + assert over == correct + assert under > correct + + +def create_trials(mocker): + return mocker.Mock( + results=[{ + 'loss': 1, + 'result': 'foo' + }] + ) + + +def test_start_calls_fmin(mocker): + mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker)) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + + args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False) + start(args) + + mock_fmin.assert_called_once() + + +def test_start_uses_mongotrials(mocker): + mock_mongotrials = mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', + return_value=create_trials(mocker)) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + + args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True) + start(args) + + mock_mongotrials.assert_called_once() + + +def test_log_results_if_loss_improves(mocker): + logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') + global CURRENT_BEST_LOSS + CURRENT_BEST_LOSS = 2 + log_results({ + 'loss': 1, + 'current_tries': 1, + 'total_tries': 2, + 'result': 'foo' + }) + + logger.assert_called_once() + + +def test_no_log_if_loss_does_not_improve(mocker): + logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') + global CURRENT_BEST_LOSS + CURRENT_BEST_LOSS = 2 + log_results({ + 'loss': 3, + }) + + assert not logger.called From a36fd00f6a76e54da1135b354f52aa39e4f93608 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 26 Dec 2017 18:39:23 +0200 Subject: [PATCH 45/64] also print dot when hyperopt eval result is fail --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index aeb7c1456..d0d0916f8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -129,6 +129,7 @@ def optimizer(params): trade_count = len(results.index) if trade_count == 0: + print('.', end='') return { 'status': STATUS_FAIL, 'loss': float('inf') From 965616b2144e1f32306253ba51f1c02c5ba0576c Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Thu, 28 Dec 2017 10:11:32 +0100 Subject: [PATCH 46/64] Update sqlalchemy from 1.1.15 to 1.2.0 (#245) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b37a15c79..9d6c3e4d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-bittrex==0.2.2 -SQLAlchemy==1.1.15 +SQLAlchemy==1.2.0 python-telegram-bot==9.0.0 arrow==0.12.0 cachetools==2.0.1 From f48f5d0f31de205e274303ed5aef14d595b7ab1b Mon Sep 17 00:00:00 2001 From: kryofly Date: Thu, 28 Dec 2017 15:58:02 +0100 Subject: [PATCH 47/64] tests for dataframe, whitelist and backtesting --- freqtrade/tests/optimize/test_backtesting.py | 104 +++++++++++++++++++ freqtrade/tests/test_acl_pair.py | 83 +++++++++++++++ freqtrade/tests/test_dataframe.py | 27 +++++ 3 files changed, 214 insertions(+) create mode 100644 freqtrade/tests/test_acl_pair.py create mode 100644 freqtrade/tests/test_dataframe.py diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 99f95d9bf..e4405fb4c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212 +import math import os import pandas as pd from freqtrade import exchange, optimize @@ -120,3 +121,106 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): if os.path.isfile(file2): os.remove(file2) + + +def trim_dataframe(df, num): + new = dict() + for pair, pair_data in df.items(): + new[pair] = pair_data[-num:] # last 50 rows + return new + +def load_data_test(what): + data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) + data = trim_dataframe(data, -40) + pair = data['BTC_UNITEST'] + + # Depending on the what parameter we now adjust the + # loaded data: + # pair :: [{'O': 0.123, 'H': 0.123, 'L': 0.123, 'C': 0.123, 'V': 123.123, 'T': '2017-11-04T23:02:00', 'BV': 0.123}] + if what == 'raise': + o = h = l = c = 0.001 + l -= 0.0001 + h += 0.0001 + for frame in pair: + o += 0.0001 + h += 0.0001 + l += 0.0001 + c += 0.0001 + o = round(o,9) # round to satoshis + h = round(h,9) + l = round(l,9) + c = round(c,9) + frame['O'] = o + frame['H'] = h + frame['L'] = l + frame['C'] = c + if what == 'lower': + o = h = l = c = 0.001 + l -= 0.0001 + h += 0.0001 + for frame in pair: + o -= 0.0001 + h -= 0.0001 + l -= 0.0001 + c -= 0.0001 + o = round(o,9) # round to satoshis + h = round(h,9) + l = round(l,9) + c = round(c,9) + frame['O'] = o + frame['H'] = h + frame['L'] = l + frame['C'] = c + if what == 'sine': + i = 0 + o = h = l = c = (2 + math.sin(i/10)) / 1000 + h += 0.0001 + l -= 0.0001 + for frame in pair: + o = (2 + math.sin(i/10)) / 1000 + h = (2 + math.sin(i/10)) / 1000 + 0.0001 + l = (2 + math.sin(i/10)) / 1000 - 0.0001 + c = (2 + math.sin(i/10)) / 1000 - 0.000001 + + o = round(o,9) # round to satoshis + h = round(h,9) + l = round(l,9) + c = round(c,9) + frame['O'] = o + frame['H'] = h + frame['L'] = l + frame['C'] = c + i += 1 + return data + +def simple_backtest(config, contour, num_results): + data = load_data_test(contour) + processed = optimize.preprocess(data) + assert isinstance(processed, dict) + results = backtest(config['stake_amount'], processed, 1, True) + # results :: + assert len(results) == num_results + +# Test backtest on offline data +# loaded by freqdata/optimize/__init__.py::load_data() + +def test_backtest(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) + results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) + num_resutls = len(results) + assert num_resutls > 0 + +def test_processed(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + data = load_data_test('raise') + processed = optimize.preprocess(data) + +def test_raise(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + tests = [['raise', 359], ['lower', 0], ['sine', 1734]] + for [contour, numres] in tests: + simple_backtest(default_conf, contour, numres) + + + diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py new file mode 100644 index 000000000..b1e57755c --- /dev/null +++ b/freqtrade/tests/test_acl_pair.py @@ -0,0 +1,83 @@ + +# whitelist, blacklist, filtering, all of that will +# eventually become some rules to run on a generic ACL engine + +# perhaps try to anticipate that by using some python package + +import pytest +from unittest.mock import MagicMock +import copy + +from freqtrade.main import refresh_whitelist +#from freqtrade.exchange import Exchanges +from freqtrade import exchange + +# "deep equal" +def assert_list_equal (l1, l2): + for pair in l1: + assert pair in l2 + for pair in l2: + assert pair in l1 + +def whitelist_conf(): + return { + "stake_currency":"BTC", + "exchange": { + "pair_whitelist": [ + "BTC_ETH", + "BTC_TKN", + "BTC_TRST", + "BTC_SWT", + "BTC_BCC" + ], + }, + } + +def get_health(): + return [{'Currency': 'ETH', + 'IsActive': True + }, + {'Currency': 'TKN', + 'IsActive': True + }] + +def get_health_empty(): + return [] + +# below three test could be merged into a single +# test that ran randomlly generated health lists + +def test_refresh_whitelist(mocker): + conf = whitelist_conf() + mocker.patch.dict('freqtrade.main._CONF', conf) + mocker.patch.multiple('freqtrade.main.exchange', + get_wallet_health=get_health) + # no argument: use the whitelist provided by config + refresh_whitelist() + whitelist = ['BTC_ETH', 'BTC_TKN'] + pairslist = conf['exchange']['pair_whitelist'] + # Ensure all except those in whitelist are removed + assert_list_equal(whitelist, pairslist) + +def test_refresh_whitelist_dynamic(mocker): + conf = whitelist_conf() + mocker.patch.dict('freqtrade.main._CONF', conf) + mocker.patch.multiple('freqtrade.main.exchange', + get_wallet_health=get_health) + # argument: use the whitelist dynamically by exchange-volume + whitelist = ['BTC_ETH', 'BTC_TKN'] + refresh_whitelist(whitelist) + pairslist = conf['exchange']['pair_whitelist'] + assert_list_equal(whitelist, pairslist) + +def test_refresh_whitelist_dynamic_empty(mocker): + conf = whitelist_conf() + mocker.patch.dict('freqtrade.main._CONF', conf) + mocker.patch.multiple('freqtrade.main.exchange', + get_wallet_health=get_health_empty) + # argument: use the whitelist dynamically by exchange-volume + whitelist = [] + conf['exchange']['pair_whitelist'] = [] + refresh_whitelist(whitelist) + pairslist = conf['exchange']['pair_whitelist'] + assert_list_equal(whitelist, pairslist) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py new file mode 100644 index 000000000..58d1474fd --- /dev/null +++ b/freqtrade/tests/test_dataframe.py @@ -0,0 +1,27 @@ + +import pytest +import pandas + +from freqtrade import analyze +import freqtrade.optimize +from pandas import DataFrame + +_pairs = ['BTC_ETH'] + +def load_dataframe_pair(pairs): + ld = freqtrade.optimize.load_data(ticker_interval=5, pairs=pairs) + assert isinstance(ld, dict) + assert isinstance(pairs[0], str) + dataframe = ld[pairs[0]] + dataframe = analyze.analyze_ticker(dataframe) + return dataframe + +def test_dataframe_load(): + dataframe = load_dataframe_pair(_pairs) + assert isinstance(dataframe, pandas.core.frame.DataFrame) + +def test_dataframe_columns_exists(): + dataframe = load_dataframe_pair(_pairs) + assert 'high' in dataframe.columns + assert 'low' in dataframe.columns + assert 'close' in dataframe.columns From ab112581a71fa42848561fe0b3893832255baf8f Mon Sep 17 00:00:00 2001 From: kryofly Date: Thu, 28 Dec 2017 20:05:33 +0100 Subject: [PATCH 48/64] tests: anal stretching to accomodate flake8 --- freqtrade/tests/optimize/test_backtesting.py | 88 ++++++++++---------- freqtrade/tests/test_acl_pair.py | 24 +++--- freqtrade/tests/test_dataframe.py | 10 +-- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e4405fb4c..528feafb1 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring,W0212 - import math import os import pandas as pd @@ -126,9 +124,10 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): def trim_dataframe(df, num): new = dict() for pair, pair_data in df.items(): - new[pair] = pair_data[-num:] # last 50 rows + new[pair] = pair_data[-num:] # last 50 rows return new + def load_data_test(what): data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) data = trim_dataframe(data, -40) @@ -136,63 +135,66 @@ def load_data_test(what): # Depending on the what parameter we now adjust the # loaded data: - # pair :: [{'O': 0.123, 'H': 0.123, 'L': 0.123, 'C': 0.123, 'V': 123.123, 'T': '2017-11-04T23:02:00', 'BV': 0.123}] + # pair :: [{'O': 0.123, 'H': 0.123, 'L': 0.123, + # 'C': 0.123, 'V': 123.123, + # 'T': '2017-11-04T23:02:00', 'BV': 0.123}] if what == 'raise': - o = h = l = c = 0.001 - l -= 0.0001 + o = 0.001 + h = 0.001 + ll = 0.001 + c = 0.001 + ll -= 0.0001 h += 0.0001 for frame in pair: o += 0.0001 h += 0.0001 - l += 0.0001 + ll += 0.0001 c += 0.0001 - o = round(o,9) # round to satoshis - h = round(h,9) - l = round(l,9) - c = round(c,9) - frame['O'] = o - frame['H'] = h - frame['L'] = l - frame['C'] = c + # save prices rounded to satoshis + frame['O'] = round(o, 9) + frame['H'] = round(h, 9) + frame['L'] = round(ll, 9) + frame['C'] = round(c, 9) if what == 'lower': - o = h = l = c = 0.001 - l -= 0.0001 + o = 0.001 + h = 0.001 + ll = 0.001 + c = 0.001 + ll -= 0.0001 h += 0.0001 for frame in pair: o -= 0.0001 h -= 0.0001 - l -= 0.0001 + ll -= 0.0001 c -= 0.0001 - o = round(o,9) # round to satoshis - h = round(h,9) - l = round(l,9) - c = round(c,9) - frame['O'] = o - frame['H'] = h - frame['L'] = l - frame['C'] = c + # save prices rounded to satoshis + frame['O'] = round(o, 9) + frame['H'] = round(h, 9) + frame['L'] = round(ll, 9) + frame['C'] = round(c, 9) if what == 'sine': i = 0 - o = h = l = c = (2 + math.sin(i/10)) / 1000 + o = (2 + math.sin(i/10)) / 1000 + h = o + ll = o + c = o h += 0.0001 - l -= 0.0001 + ll -= 0.0001 for frame in pair: - o = (2 + math.sin(i/10)) / 1000 + o = (2 + math.sin(i/10)) / 1000 h = (2 + math.sin(i/10)) / 1000 + 0.0001 - l = (2 + math.sin(i/10)) / 1000 - 0.0001 + ll = (2 + math.sin(i/10)) / 1000 - 0.0001 c = (2 + math.sin(i/10)) / 1000 - 0.000001 - o = round(o,9) # round to satoshis - h = round(h,9) - l = round(l,9) - c = round(c,9) - frame['O'] = o - frame['H'] = h - frame['L'] = l - frame['C'] = c + # save prices rounded to satoshis + frame['O'] = round(o, 9) + frame['H'] = round(h, 9) + frame['L'] = round(ll, 9) + frame['C'] = round(c, 9) i += 1 return data + def simple_backtest(config, contour, num_results): data = load_data_test(contour) processed = optimize.preprocess(data) @@ -204,23 +206,23 @@ def simple_backtest(config, contour, num_results): # Test backtest on offline data # loaded by freqdata/optimize/__init__.py::load_data() -def test_backtest(default_conf, mocker): + +def test_backtest2(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) num_resutls = len(results) assert num_resutls > 0 + def test_processed(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) data = load_data_test('raise') - processed = optimize.preprocess(data) + assert optimize.preprocess(data) + def test_raise(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) tests = [['raise', 359], ['lower', 0], ['sine', 1734]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres) - - - diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index b1e57755c..8748d97b8 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,27 +1,20 @@ +from freqtrade.main import refresh_whitelist # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine - # perhaps try to anticipate that by using some python package -import pytest -from unittest.mock import MagicMock -import copy -from freqtrade.main import refresh_whitelist -#from freqtrade.exchange import Exchanges -from freqtrade import exchange - -# "deep equal" -def assert_list_equal (l1, l2): +def assert_list_equal(l1, l2): for pair in l1: assert pair in l2 for pair in l2: assert pair in l1 + def whitelist_conf(): return { - "stake_currency":"BTC", + "stake_currency": "BTC", "exchange": { "pair_whitelist": [ "BTC_ETH", @@ -33,13 +26,15 @@ def whitelist_conf(): }, } + def get_health(): return [{'Currency': 'ETH', 'IsActive': True - }, + }, {'Currency': 'TKN', 'IsActive': True - }] + }] + def get_health_empty(): return [] @@ -47,6 +42,7 @@ def get_health_empty(): # below three test could be merged into a single # test that ran randomlly generated health lists + def test_refresh_whitelist(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) @@ -59,6 +55,7 @@ def test_refresh_whitelist(mocker): # Ensure all except those in whitelist are removed assert_list_equal(whitelist, pairslist) + def test_refresh_whitelist_dynamic(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) @@ -70,6 +67,7 @@ def test_refresh_whitelist_dynamic(mocker): pairslist = conf['exchange']['pair_whitelist'] assert_list_equal(whitelist, pairslist) + def test_refresh_whitelist_dynamic_empty(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 58d1474fd..916985406 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,13 +1,11 @@ - -import pytest import pandas from freqtrade import analyze import freqtrade.optimize -from pandas import DataFrame _pairs = ['BTC_ETH'] + def load_dataframe_pair(pairs): ld = freqtrade.optimize.load_data(ticker_interval=5, pairs=pairs) assert isinstance(ld, dict) @@ -16,12 +14,14 @@ def load_dataframe_pair(pairs): dataframe = analyze.analyze_ticker(dataframe) return dataframe + def test_dataframe_load(): dataframe = load_dataframe_pair(_pairs) assert isinstance(dataframe, pandas.core.frame.DataFrame) + def test_dataframe_columns_exists(): dataframe = load_dataframe_pair(_pairs) - assert 'high' in dataframe.columns - assert 'low' in dataframe.columns + assert 'high' in dataframe.columns + assert 'low' in dataframe.columns assert 'close' in dataframe.columns From 847dde0d6517ecb6b1895cebefe2359c95aea8f1 Mon Sep 17 00:00:00 2001 From: kryofly Date: Fri, 29 Dec 2017 00:00:30 +0100 Subject: [PATCH 49/64] execute sell if get_signal OR ROI reached --- freqtrade/main.py | 15 +++++++++------ freqtrade/tests/test_main.py | 37 ++++++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 77f956aba..ad7cc7289 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -179,17 +179,20 @@ def handle_trade(trade: Trade) -> bool: current_rate = exchange.get_ticker(trade.pair)['bid'] # Check if minimal roi has been reached - if not min_roi_reached(trade, current_rate, datetime.utcnow()): - return False + if min_roi_reached(trade, current_rate, datetime.utcnow()): + logger.debug('Executing sell due to ROI ...') + execute_sell(trade, current_rate) + return True # Check if sell signal has been enabled and triggered if _CONF.get('experimental', {}).get('use_sell_signal'): logger.debug('Checking sell_signal ...') - if not get_signal(trade.pair, SignalType.SELL): - return False + if get_signal(trade.pair, SignalType.SELL): + logger.debug('Executing sell due to sell signal ...') + execute_sell(trade, current_rate) + return True - execute_sell(trade, current_rate) - return True + return False def get_target_bid(ticker: Dict[str, float]) -> float: diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 037d6f836..2d5263159 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -216,8 +216,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.calc_profit() == 0.00006217 assert trade.close_date is not None - -def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -235,10 +234,44 @@ def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker trade = Trade.query.first() trade.is_open = True + # FIX: sniffing logs, suggest handle_trade should not execute_sell + # instead that responsibility should be moved out of handle_trade(), + # we might just want to check if we are in a sell condition without + # executing + # if ROI is reached we must sell + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: False) + assert handle_trade(trade) + assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples + # if ROI is reached we must sell even if sell-signal is not signalled + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + assert handle_trade(trade) + assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples + +def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): + default_conf.update({'experimental': {'use_sell_signal': True}}) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch('freqtrade.main.min_roi_reached', return_value=False) + + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) + + trade = Trade.query.first() + trade.is_open = True + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: False) value_returned = handle_trade(trade) assert ('freqtrade', logging.DEBUG, 'Checking sell_signal ...') in caplog.record_tuples assert value_returned is False + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + assert handle_trade(trade) + assert ('freqtrade', logging.DEBUG, 'Executing sell due to sell signal ...') in caplog.record_tuples def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mocker): From 0d605d2396920d171d477336c0ee111cafa4936e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 28 Dec 2017 00:11:52 -0800 Subject: [PATCH 50/64] Refactor Optimize tests, and add more unit tests --- freqtrade/optimize/__init__.py | 14 +- freqtrade/tests/optimize/test_backtesting.py | 76 +-------- freqtrade/tests/optimize/test_optimize.py | 166 +++++++++++++++++++ freqtrade/tests/test_main.py | 2 +- 4 files changed, 175 insertions(+), 83 deletions(-) create mode 100644 freqtrade/tests/optimize/test_optimize.py diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index ac077506a..1be7ce536 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -87,17 +87,17 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool: )) filepair = pair.replace("-", "_") - filename = os.path.join(path, '{}-{}.json'.format( - filepair, - interval, + filename = os.path.join(path, '{pair}-{interval}.json'.format( + pair=filepair, + interval=interval, )) filename = filename.replace('USDT_BTC', 'BTC_FAKEBULL') if os.path.isfile(filename): with open(filename, "rt") as fp: data = json.load(fp) - logger.debug("Current Start:", data[1]['T']) - logger.debug("Current End: ", data[-1:][0]['T']) + logger.debug("Current Start: {}".format(data[1]['T'])) + logger.debug("Current End: {}".format(data[-1:][0]['T'])) else: data = [] logger.debug("Current Start: None") @@ -107,8 +107,8 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool: for row in new_data: if row not in data: data.append(row) - logger.debug("New Start:", data[1]['T']) - logger.debug("New End: ", data[-1:][0]['T']) + logger.debug("New Start: {}".format(data[1]['T'])) + logger.debug("New End: {}".format(data[-1:][0]['T'])) data = sorted(data, key=lambda data: data['T']) with open(filename, "wt") as fp: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 99f95d9bf..e3d423b7d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,11 +1,9 @@ # pragma pylint: disable=missing-docstring,W0212 -import os import pandas as pd from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe -from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata def test_generate_text_table(): @@ -40,7 +38,7 @@ def test_backtest(default_conf, mocker): assert not results.empty -def test_1min_ticker_interval(default_conf, mocker): +def test_backtest_1min_ticker_interval(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -48,75 +46,3 @@ def test_1min_ticker_interval(default_conf, mocker): data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 1, True) assert not results.empty - - -def test_backtest_with_new_pair(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) - - optimize.load_data(ticker_interval=1, pairs=['BTC_MEME']) - file = 'freqtrade/tests/testdata/BTC_MEME-1.json' - assert os.path.isfile(file) is True - - # delete file freshly downloaded - if os.path.isfile(file): - os.remove(file) - - -def test_testdata_path(): - assert os.path.join('freqtrade', 'tests', 'testdata') in testdata_path() - - -def test_download_pairs(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' - file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' - file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json' - file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json' - - assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True - - assert os.path.isfile(file1_1) is True - assert os.path.isfile(file1_5) is True - assert os.path.isfile(file2_1) is True - assert os.path.isfile(file2_5) is True - - # delete files freshly downloaded - if os.path.isfile(file1_1): - os.remove(file1_1) - - if os.path.isfile(file1_5): - os.remove(file1_5) - - if os.path.isfile(file2_1): - os.remove(file2_1) - - if os.path.isfile(file2_5): - os.remove(file2_5) - - -def test_download_backtesting_testdata(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - # Download a 1 min ticker file - file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' - download_backtesting_testdata(pair="BTC-XEL", interval=1) - assert os.path.isfile(file1) is True - - if os.path.isfile(file1): - os.remove(file1) - - # Download a 5 min ticker file - file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' - download_backtesting_testdata(pair="BTC-STORJ", interval=5) - assert os.path.isfile(file2) is True - - if os.path.isfile(file2): - os.remove(file2) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py new file mode 100644 index 000000000..aad20567e --- /dev/null +++ b/freqtrade/tests/optimize/test_optimize.py @@ -0,0 +1,166 @@ +# pragma pylint: disable=missing-docstring,W0212 + +import os +import logging +from shutil import copyfile +from freqtrade import exchange, optimize +from freqtrade.exchange import Bittrex +from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata + + +def _backup_file(file: str, copy_file: bool = False) -> None: + """ + Backup existing file to avoid deleting the user file + :param file: complete path to the file + :param touch_file: create an empty file in replacement + :return: None + """ + file_swp = file + '.swp' + if os.path.isfile(file): + os.rename(file, file_swp) + + if copy_file: + copyfile(file_swp, file) + + +def _clean_test_file(file: str) -> None: + """ + Backup existing file to avoid deleting the user file + :param file: complete path to the file + :return: None + """ + file_swp = file + '.swp' + # 1. Delete file from the test + if os.path.isfile(file): + os.remove(file) + + # 2. Rollback to the initial file + if os.path.isfile(file_swp): + os.rename(file_swp, file) + + +def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog): + mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + + exchange._API = Bittrex({'key': '', 'secret': ''}) + + file = 'freqtrade/tests/testdata/BTC_ETH-5.json' + _backup_file(file, copy_file=True) + optimize.load_data(pairs=['BTC_ETH']) + assert os.path.isfile(file) is True + assert ('freqtrade.optimize', + logging.INFO, + 'Download the pair: "BTC_ETH", Interval: 5 min' + ) not in caplog.record_tuples + _clean_test_file(file) + + +def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog): + mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + + exchange._API = Bittrex({'key': '', 'secret': ''}) + + file = 'freqtrade/tests/testdata/BTC_ETH-1.json' + _backup_file(file, copy_file=True) + optimize.load_data(ticker_interval=1, pairs=['BTC_ETH']) + assert os.path.isfile(file) is True + assert ('freqtrade.optimize', + logging.INFO, + 'Download the pair: "BTC_ETH", Interval: 1 min' + ) not in caplog.record_tuples + _clean_test_file(file) + + +def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, caplog): + mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + + exchange._API = Bittrex({'key': '', 'secret': ''}) + + file = 'freqtrade/tests/testdata/BTC_MEME-1.json' + _backup_file(file) + optimize.load_data(ticker_interval=1, pairs=['BTC_MEME']) + assert os.path.isfile(file) is True + assert ('freqtrade.optimize', + logging.INFO, + 'Download the pair: "BTC_MEME", Interval: 1 min' + ) in caplog.record_tuples + _clean_test_file(file) + + +def test_testdata_path(): + assert os.path.join('freqtrade', 'tests', 'testdata') in testdata_path() + + +def test_download_pairs(default_conf, ticker_history, mocker): + mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + exchange._API = Bittrex({'key': '', 'secret': ''}) + + file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' + file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' + file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json' + file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json' + + _backup_file(file1_1) + _backup_file(file1_5) + _backup_file(file2_1) + _backup_file(file2_5) + + assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True + + assert os.path.isfile(file1_1) is True + assert os.path.isfile(file1_5) is True + assert os.path.isfile(file2_1) is True + assert os.path.isfile(file2_5) is True + + # clean files freshly downloaded + _clean_test_file(file1_1) + _clean_test_file(file1_5) + _clean_test_file(file2_1) + _clean_test_file(file2_5) + + +def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog): + mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', + side_effect=BaseException('File Error')) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + exchange._API = Bittrex({'key': '', 'secret': ''}) + + file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' + file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' + _backup_file(file1_1) + _backup_file(file1_5) + + download_pairs(pairs=['BTC-MEME']) + # clean files freshly downloaded + _clean_test_file(file1_1) + _clean_test_file(file1_5) + assert ('freqtrade.optimize.__init__', + logging.INFO, + 'Failed to download the pair: "BTC-MEME", Interval: 1 min' + ) in caplog.record_tuples + + +def test_download_backtesting_testdata(default_conf, ticker_history, mocker): + mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + exchange._API = Bittrex({'key': '', 'secret': ''}) + + # Download a 1 min ticker file + file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' + _backup_file(file1) + download_backtesting_testdata(pair="BTC-XEL", interval=1) + assert os.path.isfile(file1) is True + _clean_test_file(file1) + + # Download a 5 min ticker file + file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' + _backup_file(file2) + + download_backtesting_testdata(pair="BTC-STORJ", interval=5) + assert os.path.isfile(file2) is True + _clean_test_file(file2) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 037d6f836..f8bad9fa5 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -217,7 +217,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.close_date is not None -def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_experimental(default_conf, ticker, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) From 3e0458da7dd61cd09e92ab8bd344707422bf07f0 Mon Sep 17 00:00:00 2001 From: kryofly Date: Fri, 29 Dec 2017 09:40:24 +0100 Subject: [PATCH 51/64] flake8 --- freqtrade/tests/test_main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 2d5263159..3d6335572 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -216,6 +216,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.calc_profit() == 0.00006217 assert trade.close_date is not None + def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -247,6 +248,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog) assert handle_trade(trade) assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples + def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -271,7 +273,8 @@ def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker assert value_returned is False mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) assert handle_trade(trade) - assert ('freqtrade', logging.DEBUG, 'Executing sell due to sell signal ...') in caplog.record_tuples + s = 'Executing sell due to sell signal ...' + assert ('freqtrade', logging.DEBUG, s) in caplog.record_tuples def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mocker): From 37613fc056705e3fa6435a630cf922bac4fa7a67 Mon Sep 17 00:00:00 2001 From: kryofly Date: Fri, 29 Dec 2017 17:53:58 +0100 Subject: [PATCH 52/64] flake8 --- freqtrade/tests/optimize/test_backtesting.py | 73 -------------------- 1 file changed, 73 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 3acd4b142..0d153a22e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212 import math -import os import pandas as pd from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex @@ -50,78 +49,6 @@ def test_backtest_1min_ticker_interval(default_conf, mocker): assert not results.empty -def test_backtest_with_new_pair(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) - - optimize.load_data(ticker_interval=1, pairs=['BTC_MEME']) - file = 'freqtrade/tests/testdata/BTC_MEME-1.json' - assert os.path.isfile(file) is True - - # delete file freshly downloaded - if os.path.isfile(file): - os.remove(file) - - -def test_testdata_path(): - assert os.path.join('freqtrade', 'tests', 'testdata') in testdata_path() - - -def test_download_pairs(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' - file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' - file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json' - file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json' - - assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True - - assert os.path.isfile(file1_1) is True - assert os.path.isfile(file1_5) is True - assert os.path.isfile(file2_1) is True - assert os.path.isfile(file2_5) is True - - # delete files freshly downloaded - if os.path.isfile(file1_1): - os.remove(file1_1) - - if os.path.isfile(file1_5): - os.remove(file1_5) - - if os.path.isfile(file2_1): - os.remove(file2_1) - - if os.path.isfile(file2_5): - os.remove(file2_5) - - -def test_download_backtesting_testdata(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - # Download a 1 min ticker file - file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' - download_backtesting_testdata(pair="BTC-XEL", interval=1) - assert os.path.isfile(file1) is True - - if os.path.isfile(file1): - os.remove(file1) - - # Download a 5 min ticker file - file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' - download_backtesting_testdata(pair="BTC-STORJ", interval=5) - assert os.path.isfile(file2) is True - - if os.path.isfile(file2): - os.remove(file2) - - def trim_dataframe(df, num): new = dict() for pair, pair_data in df.items(): From c8c8c626b0cfa4519555f213b3c4e12f18f4f2e8 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 29 Dec 2017 19:52:55 -0800 Subject: [PATCH 53/64] Fix issue #248: missing configuration when executing /forcesell This is not a beautiful workaround, I am not proud of it, but a redesigning of main.py and telegram.py will be necessary for a better integration. Any better solution is welcome. --- README.md | 2 +- freqtrade/main.py | 41 +++++++++----- freqtrade/misc.py | 1 + freqtrade/tests/test_main.py | 103 ++++++++++++++++++++++++++++++++++- 4 files changed, 131 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 286591064..57000301f 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. -`fiat_currency` set the fiat to use for the conversion form coin to +`fiat_display_currency` set the fiat to use for the conversion form coin to fiat in Telegram. The valid value are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", diff --git a/freqtrade/main.py b/freqtrade/main.py index ad7cc7289..66e54f237 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -124,26 +124,39 @@ def execute_sell(trade: Trade, limit: float) -> None: fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) profit_trade = trade.calc_profit(rate=limit) - fiat_converter = CryptoToFiatConverter() - profit_fiat = fiat_converter.convert_amount( - profit_trade, - _CONF['stake_currency'], - _CONF['fiat_display_currency'] - ) + message = '*{exchange}:* Selling [{pair}]({pair_url}) with limit `{limit:.8f}`'.format( + exchange=trade.exchange, + pair=trade.pair.replace('_', '/'), + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit + ) - rpc.send_msg('*{exchange}:* Selling [{pair}]({pair_url}) with limit `{limit:.8f}`' - '` (profit: ~{profit_percent:.2f}%, {profit_coin:.8f} {coin}`' - '` / {profit_fiat:.3f} {fiat})`'.format( - exchange=trade.exchange, - pair=trade.pair.replace('_', '/'), - pair_url=exchange.get_pair_detail_url(trade.pair), - limit=limit, + # For regular case, when the configuration exists + if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF: + fiat_converter = CryptoToFiatConverter() + profit_fiat = fiat_converter.convert_amount( + profit_trade, + _CONF['stake_currency'], + _CONF['fiat_display_currency'] + ) + message += '` (profit: ~{profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ + '` / {profit_fiat:.3f} {fiat})`'.format( profit_percent=fmt_exp_profit, profit_coin=profit_trade, coin=_CONF['stake_currency'], profit_fiat=profit_fiat, fiat=_CONF['fiat_display_currency'], - )) + ) + # Because telegram._forcesell does not have the configuration + # Ignore the FIAT value and does not show the stake_currency as well + else: + message += '` (profit: ~{profit_percent:.2f}%, {profit_coin:.8f})`'.format( + profit_percent=fmt_exp_profit, + profit_coin=profit_trade + ) + + # Send the message + rpc.send_msg(message) Trade.session.flush() diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 57e7c6735..5196d2ae7 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -280,6 +280,7 @@ CONF_SCHEMA = { 'max_open_trades', 'stake_currency', 'stake_amount', + 'fiat_display_currency', 'dry_run', 'minimal_roi', 'bid_strategy', diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 3d6335572..946deeec5 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -11,7 +11,7 @@ from freqtrade import DependencyException, OperationalException from freqtrade.analyze import SignalType from freqtrade.exchange import Exchanges from freqtrade.main import create_trade, handle_trade, init, \ - get_target_bid, _process + get_target_bid, _process, execute_sell from freqtrade.misc import get_state, State from freqtrade.persistence import Trade @@ -314,3 +314,104 @@ def test_balance_fully_last_side(mocker): def test_balance_bigger_last_ask(mocker): mocker.patch.dict('freqtrade.main._CONF', {'bid_strategy': {'ask_last_balance': 1.0}}) assert get_target_bid({'ask': 5, 'last': 10}) == 5 + + +def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch('freqtrade.rpc.init', MagicMock()) + rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) + init(default_conf, create_engine('sqlite://')) + + # Create some test data + create_trade(0.001) + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up) + + execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: ~6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch('freqtrade.rpc.init', MagicMock()) + rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) + mocker.patch.multiple('freqtrade.rpc.telegram', + _CONF=default_conf, + init=MagicMock(), + send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) + init(default_conf, create_engine('sqlite://')) + + # Create some test data + create_trade(0.001) + + trade = Trade.query.first() + assert trade + + # Decrease the price and sell it + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down) + + execute_sell(trade=trade, limit=ticker_sell_down()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: ~-5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_without_conf(default_conf, ticker, ticker_sell_up, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch('freqtrade.rpc.init', MagicMock()) + rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker) + init(default_conf, create_engine('sqlite://')) + + # Create some test data + create_trade(0.001) + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up) + mocker.patch('freqtrade.main._CONF', {}) + + execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert '(profit: ~6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] + assert 'USD' not in rpc_mock.call_args_list[-1][0][0] From e81a9cbb17d33fff15381638e5040a8562a9bb2c Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 29 Dec 2017 23:12:52 -0800 Subject: [PATCH 54/64] Increase code coverage Change log: * Increase code coverage for test_exchange.py * Move Exchange Unit tests files tests/exchange/ * Move RPC Unit tests files tests/rpc/ --- freqtrade/tests/exchange/test_exchange.py | 188 ++++++++++++++++++ .../{ => exchange}/test_exchange_bittrex.py | 0 freqtrade/tests/{ => rpc}/test_rpc.py | 0 .../tests/{ => rpc}/test_rpc_telegram.py | 0 freqtrade/tests/test_exchange.py | 36 ---- 5 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 freqtrade/tests/exchange/test_exchange.py rename freqtrade/tests/{ => exchange}/test_exchange_bittrex.py (100%) rename freqtrade/tests/{ => rpc}/test_rpc.py (100%) rename freqtrade/tests/{ => rpc}/test_rpc_telegram.py (100%) delete mode 100644 freqtrade/tests/test_exchange.py diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py new file mode 100644 index 000000000..2da657642 --- /dev/null +++ b/freqtrade/tests/exchange/test_exchange.py @@ -0,0 +1,188 @@ +# pragma pylint: disable=missing-docstring,C0103 +from unittest.mock import MagicMock +from requests.exceptions import RequestException +from random import randint +import logging +import pytest + +from freqtrade import OperationalException +from freqtrade.exchange import init, validate_pairs, buy, sell, get_balance, get_balances, \ + get_ticker, cancel_order, get_name, get_fee + + +def test_init(default_conf, mocker, caplog): + mocker.patch('freqtrade.exchange.validate_pairs', side_effect=lambda s: True) + init(config=default_conf) + assert ('freqtrade.exchange', + logging.INFO, + 'Instance is running with dry_run enabled' + ) in caplog.record_tuples + + +def test_init_exception(default_conf, mocker): + default_conf['exchange']['name'] = 'wrong_exchange_name' + + with pytest.raises( + OperationalException, + match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + init(config=default_conf) + + +def test_validate_pairs(default_conf, mocker): + api_mock = MagicMock() + api_mock.get_markets = MagicMock(return_value=[ + 'BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT', 'BTC_BCC', + ]) + mocker.patch('freqtrade.exchange._API', api_mock) + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + validate_pairs(default_conf['exchange']['pair_whitelist']) + + +def test_validate_pairs_not_available(default_conf, mocker): + api_mock = MagicMock() + api_mock.get_markets = MagicMock(return_value=[]) + mocker.patch('freqtrade.exchange._API', api_mock) + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + with pytest.raises(OperationalException, match=r'not available'): + validate_pairs(default_conf['exchange']['pair_whitelist']) + + +def test_validate_pairs_not_compatible(default_conf, mocker): + api_mock = MagicMock() + api_mock.get_markets = MagicMock(return_value=['BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT']) + default_conf['stake_currency'] = 'ETH' + mocker.patch('freqtrade.exchange._API', api_mock) + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + with pytest.raises(OperationalException, match=r'not compatible'): + validate_pairs(default_conf['exchange']['pair_whitelist']) + + +def test_validate_pairs_exception(default_conf, mocker, caplog): + api_mock = MagicMock() + api_mock.get_markets = MagicMock(side_effect=RequestException()) + mocker.patch('freqtrade.exchange._API', api_mock) + + # with pytest.raises(RequestException, match=r'Unable to validate pairs'): + validate_pairs(default_conf['exchange']['pair_whitelist']) + assert ('freqtrade.exchange', + logging.WARNING, + 'Unable to validate pairs (assuming they are correct). Reason: ' + ) in caplog.record_tuples + + +def test_buy_dry_run(default_conf, mocker): + default_conf['dry_run'] = True + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert 'dry_run_buy_' in buy(pair='BTC_ETH', rate=200, amount=1) + + +def test_buy_prod(default_conf, mocker): + api_mock = MagicMock() + api_mock.buy = MagicMock(return_value='dry_run_buy_{}'.format(randint(0, 10**6))) + mocker.patch('freqtrade.exchange._API', api_mock) + + default_conf['dry_run'] = False + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert 'dry_run_buy_' in buy(pair='BTC_ETH', rate=200, amount=1) + + +def test_sell_dry_run(default_conf, mocker): + default_conf['dry_run'] = True + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert 'dry_run_sell_' in sell(pair='BTC_ETH', rate=200, amount=1) + + +def test_sell_prod(default_conf, mocker): + api_mock = MagicMock() + api_mock.sell = MagicMock(return_value='dry_run_sell_{}'.format(randint(0, 10**6))) + mocker.patch('freqtrade.exchange._API', api_mock) + + default_conf['dry_run'] = False + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert 'dry_run_sell_' in sell(pair='BTC_ETH', rate=200, amount=1) + + +def test_get_balance_dry_run(default_conf, mocker): + default_conf['dry_run'] = True + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert get_balance(currency='BTC') == 999.9 + + +def test_get_balance_prod(default_conf, mocker): + api_mock = MagicMock() + api_mock.get_balance = MagicMock(return_value=123.4) + mocker.patch('freqtrade.exchange._API', api_mock) + + default_conf['dry_run'] = False + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert get_balance(currency='BTC') == 123.4 + + +def test_get_balances_dry_run(default_conf, mocker): + default_conf['dry_run'] = True + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert get_balances() == [] + + +def test_get_balances_prod(default_conf, mocker): + balance_item = { + 'Currency': '1ST', + 'Balance': 10.0, + 'Available': 10.0, + 'Pending': 0.0, + 'CryptoAddress': None + } + + api_mock = MagicMock() + api_mock.get_balances = MagicMock(return_value=[balance_item, balance_item, balance_item]) + mocker.patch('freqtrade.exchange._API', api_mock) + + default_conf['dry_run'] = False + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert len(get_balances()) == 3 + assert get_balances()[0]['Currency'] == '1ST' + assert get_balances()[0]['Balance'] == 10.0 + assert get_balances()[0]['Available'] == 10.0 + assert get_balances()[0]['Pending'] == 0.0 + + +def test_get_ticker(mocker, ticker): + + api_mock = MagicMock() + api_mock.get_ticker = MagicMock(return_value=ticker()) + mocker.patch('freqtrade.exchange._API', api_mock) + + ticker = get_ticker(pair='BTC_ETH') + assert ticker['bid'] == 0.00001098 + assert ticker['ask'] == 0.00001099 + assert ticker['bid'] == 0.00001098 + + +def test_cancel_order_dry_run(default_conf, mocker): + default_conf['dry_run'] = True + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + + assert cancel_order(order_id='123') is None + + +def test_get_name(default_conf, mocker): + mocker.patch('freqtrade.exchange.validate_pairs', side_effect=lambda s: True) + default_conf['exchange']['name'] = 'bittrex' + init(default_conf) + + assert get_name() == 'Bittrex' + + +def test_get_fee(default_conf, mocker): + mocker.patch('freqtrade.exchange.validate_pairs', side_effect=lambda s: True) + init(default_conf) + + assert get_fee() == 0.0025 diff --git a/freqtrade/tests/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py similarity index 100% rename from freqtrade/tests/test_exchange_bittrex.py rename to freqtrade/tests/exchange/test_exchange_bittrex.py diff --git a/freqtrade/tests/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py similarity index 100% rename from freqtrade/tests/test_rpc.py rename to freqtrade/tests/rpc/test_rpc.py diff --git a/freqtrade/tests/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py similarity index 100% rename from freqtrade/tests/test_rpc_telegram.py rename to freqtrade/tests/rpc/test_rpc_telegram.py diff --git a/freqtrade/tests/test_exchange.py b/freqtrade/tests/test_exchange.py deleted file mode 100644 index 78bb89bfc..000000000 --- a/freqtrade/tests/test_exchange.py +++ /dev/null @@ -1,36 +0,0 @@ -# pragma pylint: disable=missing-docstring,C0103 -from unittest.mock import MagicMock - -import pytest - -from freqtrade import OperationalException -from freqtrade.exchange import validate_pairs - - -def test_validate_pairs(default_conf, mocker): - api_mock = MagicMock() - api_mock.get_markets = MagicMock(return_value=[ - 'BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT', 'BTC_BCC', - ]) - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - validate_pairs(default_conf['exchange']['pair_whitelist']) - - -def test_validate_pairs_not_available(default_conf, mocker): - api_mock = MagicMock() - api_mock.get_markets = MagicMock(return_value=[]) - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - with pytest.raises(OperationalException, match=r'not available'): - validate_pairs(default_conf['exchange']['pair_whitelist']) - - -def test_validate_pairs_not_compatible(default_conf, mocker): - api_mock = MagicMock() - api_mock.get_markets = MagicMock(return_value=['BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT']) - default_conf['stake_currency'] = 'ETH' - mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - with pytest.raises(OperationalException, match=r'not compatible'): - validate_pairs(default_conf['exchange']['pair_whitelist']) From f7398e615add117521655ef138d2a8fc79d718e9 Mon Sep 17 00:00:00 2001 From: kryofly <34599184+kryofly@users.noreply.github.com> Date: Sat, 30 Dec 2017 11:55:23 +0100 Subject: [PATCH 55/64] Improve backtesting tests (#256) * test bugfix dataframe trimming * flake8 (as usual) * tests backtesting cleanup and bugfix * flake8 * test backtesting::start() * tests cleanup set() usage * tests: add missing assert --- freqtrade/tests/optimize/test_backtesting.py | 131 ++++++++++--------- freqtrade/tests/test_acl_pair.py | 13 +- 2 files changed, 70 insertions(+), 74 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 0d153a22e..94c062cac 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -2,9 +2,11 @@ import math import pandas as pd +# from unittest.mock import MagicMock from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe +# import freqtrade.optimize.backtesting as backtesting def test_generate_text_table(): @@ -49,77 +51,53 @@ def test_backtest_1min_ticker_interval(default_conf, mocker): assert not results.empty -def trim_dataframe(df, num): - new = dict() - for pair, pair_data in df.items(): - new[pair] = pair_data[-num:] # last 50 rows +def trim_dictlist(dl, num): + new = {} + for pair, pair_data in dl.items(): + # Can't figure out why -num wont work + new[pair] = pair_data[num:] return new def load_data_test(what): data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) - data = trim_dataframe(data, -40) + data = trim_dictlist(data, -100) pair = data['BTC_UNITEST'] - + datalen = len(pair) # Depending on the what parameter we now adjust the - # loaded data: + # loaded data looks: # pair :: [{'O': 0.123, 'H': 0.123, 'L': 0.123, # 'C': 0.123, 'V': 123.123, # 'T': '2017-11-04T23:02:00', 'BV': 0.123}] + base = 0.001 if what == 'raise': - o = 0.001 - h = 0.001 - ll = 0.001 - c = 0.001 - ll -= 0.0001 - h += 0.0001 - for frame in pair: - o += 0.0001 - h += 0.0001 - ll += 0.0001 - c += 0.0001 - # save prices rounded to satoshis - frame['O'] = round(o, 9) - frame['H'] = round(h, 9) - frame['L'] = round(ll, 9) - frame['C'] = round(c, 9) + return {'BTC_UNITEST': + [{'T': pair[x]['T'], # Keep old dates + 'V': pair[x]['V'], # Keep old volume + 'BV': pair[x]['BV'], # keep too + 'O': x * base, # But replace O,H,L,C + 'H': x * base + 0.0001, + 'L': x * base - 0.0001, + 'C': x * base} for x in range(0, datalen)]} if what == 'lower': - o = 0.001 - h = 0.001 - ll = 0.001 - c = 0.001 - ll -= 0.0001 - h += 0.0001 - for frame in pair: - o -= 0.0001 - h -= 0.0001 - ll -= 0.0001 - c -= 0.0001 - # save prices rounded to satoshis - frame['O'] = round(o, 9) - frame['H'] = round(h, 9) - frame['L'] = round(ll, 9) - frame['C'] = round(c, 9) + return {'BTC_UNITEST': + [{'T': pair[x]['T'], # Keep old dates + 'V': pair[x]['V'], # Keep old volume + 'BV': pair[x]['BV'], # keep too + 'O': 1 - x * base, # But replace O,H,L,C + 'H': 1 - x * base + 0.0001, + 'L': 1 - x * base - 0.0001, + 'C': 1 - x * base} for x in range(0, datalen)]} if what == 'sine': - i = 0 - o = (2 + math.sin(i/10)) / 1000 - h = o - ll = o - c = o - h += 0.0001 - ll -= 0.0001 - for frame in pair: - o = (2 + math.sin(i/10)) / 1000 - h = (2 + math.sin(i/10)) / 1000 + 0.0001 - ll = (2 + math.sin(i/10)) / 1000 - 0.0001 - c = (2 + math.sin(i/10)) / 1000 - 0.000001 - - # save prices rounded to satoshis - frame['O'] = round(o, 9) - frame['H'] = round(h, 9) - frame['L'] = round(ll, 9) - frame['C'] = round(c, 9) - i += 1 + hz = 0.1 # frequency + return {'BTC_UNITEST': + [{'T': pair[x]['T'], # Keep old dates + 'V': pair[x]['V'], # Keep old volume + 'BV': pair[x]['BV'], # keep too + 'O': math.sin(x*hz) / 1000 + base, # But replace O,H,L,C + 'H': math.sin(x*hz) / 1000 + base + 0.0001, + 'L': math.sin(x*hz) / 1000 + base - 0.0001, + 'C': math.sin(x*hz) / 1000 + base} for x in range(0, datalen)]} return data @@ -131,6 +109,7 @@ def simple_backtest(config, contour, num_results): # results :: assert len(results) == num_results + # Test backtest on offline data # loaded by freqdata/optimize/__init__.py::load_data() @@ -139,18 +118,42 @@ def test_backtest2(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) - num_resutls = len(results) - assert num_resutls > 0 + assert not results.empty def test_processed(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) - data = load_data_test('raise') - assert optimize.preprocess(data) + dict_of_tickerrows = load_data_test('raise') + dataframes = optimize.preprocess(dict_of_tickerrows) + dataframe = dataframes['BTC_UNITEST'] + cols = dataframe.columns + # assert the dataframe got some of the indicator columns + for col in ['close', 'high', 'low', 'open', 'date', + 'ema50', 'ao', 'macd', 'plus_dm']: + assert col in cols -def test_raise(default_conf, mocker): +def test_backtest_pricecontours(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) - tests = [['raise', 359], ['lower', 0], ['sine', 1734]] + tests = [['raise', 17], ['lower', 0], ['sine', 17]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres) + +# Please make this work, the load_config needs to be mocked +# and cleanups. +# def test_backtest_start(default_conf, mocker): +# default_conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] +# mocker.patch.dict('freqtrade.main._CONF', default_conf) +# # see https://pypi.python.org/pypi/pytest-mock/ +# # and http://www.voidspace.org.uk/python/mock/patch.html +# # No usage example of simple function mocking, +# # and no documentation of side_effect +# mocker.patch('freqtrade.misc.load_config', new=lambda s, t: {}) +# args = MagicMock() +# args.level = 10 +# #load_config('foo') +# backtesting.start(args) +# +# Check what sideeffect backtstesting has done. +# Probably need to capture standard-output and +# check for the generated report table. diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 8748d97b8..3cbc9cfa0 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -5,13 +5,6 @@ from freqtrade.main import refresh_whitelist # perhaps try to anticipate that by using some python package -def assert_list_equal(l1, l2): - for pair in l1: - assert pair in l2 - for pair in l2: - assert pair in l1 - - def whitelist_conf(): return { "stake_currency": "BTC", @@ -53,7 +46,7 @@ def test_refresh_whitelist(mocker): whitelist = ['BTC_ETH', 'BTC_TKN'] pairslist = conf['exchange']['pair_whitelist'] # Ensure all except those in whitelist are removed - assert_list_equal(whitelist, pairslist) + assert set(whitelist) == set(pairslist) def test_refresh_whitelist_dynamic(mocker): @@ -65,7 +58,7 @@ def test_refresh_whitelist_dynamic(mocker): whitelist = ['BTC_ETH', 'BTC_TKN'] refresh_whitelist(whitelist) pairslist = conf['exchange']['pair_whitelist'] - assert_list_equal(whitelist, pairslist) + assert set(whitelist) == set(pairslist) def test_refresh_whitelist_dynamic_empty(mocker): @@ -78,4 +71,4 @@ def test_refresh_whitelist_dynamic_empty(mocker): conf['exchange']['pair_whitelist'] = [] refresh_whitelist(whitelist) pairslist = conf['exchange']['pair_whitelist'] - assert_list_equal(whitelist, pairslist) + assert set(whitelist) == set(pairslist) From 8411844d7ef930687b4c738f797c3813bcd748b6 Mon Sep 17 00:00:00 2001 From: jblestang Date: Sat, 30 Dec 2017 14:15:07 +0100 Subject: [PATCH 56/64] Implement pair_blacklist functionality (#257) * Adding an optional black_list of pairs not to be traded * applying the blacklist also when not using --dynamic-whitelist * fix error retrieving pair in conf * Refactoring the handling of whitelist among the various functions * unit test to verify that black listed pairs are being removed from the pair_whitelist * Fixing newly added unit tests in develop * fixing flake8 code review * fix code review from @garcq --- freqtrade/main.py | 38 ++++++++++++++------------------ freqtrade/misc.py | 8 +++++++ freqtrade/tests/test_acl_pair.py | 11 ++++----- freqtrade/tests/test_main.py | 17 ++++++++++++++ 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index ad7cc7289..18bc04e6f 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -26,19 +26,17 @@ logger = logging.getLogger('freqtrade') _CONF = {} -def refresh_whitelist(whitelist: Optional[List[str]] = None) -> None: +def refresh_whitelist(whitelist: List[str]) -> List[str]: """ Check wallet health and remove pair from whitelist if necessary - :param whitelist: a new whitelist (optional) - :return: None + :param whitelist: the pair the user might want to trade + :return: the list of pairs the user wants to trade without the one unavailable or black_listed """ - whitelist = whitelist or _CONF['exchange']['pair_whitelist'] - sanitized_whitelist = [] health = exchange.get_wallet_health() for status in health: pair = '{}_{}'.format(_CONF['stake_currency'], status['Currency']) - if pair not in whitelist: + if pair not in whitelist or pair in _CONF['exchange'].get('pair_blacklist', []): continue if status['IsActive']: sanitized_whitelist.append(pair) @@ -47,27 +45,29 @@ def refresh_whitelist(whitelist: Optional[List[str]] = None) -> None: 'Ignoring %s from whitelist (reason: %s).', pair, status.get('Notice') or 'wallet is not active' ) - if _CONF['exchange']['pair_whitelist'] != sanitized_whitelist: - logger.debug('Using refreshed pair whitelist: %s ...', sanitized_whitelist) - _CONF['exchange']['pair_whitelist'] = sanitized_whitelist + return sanitized_whitelist -def _process(dynamic_whitelist: Optional[int] = 0) -> bool: +def _process(nb_assets: Optional[int] = 0) -> bool: """ Queries the persistence layer for open trades and handles them, otherwise a new trade is created. - :param: dynamic_whitelist: True is a dynamic whitelist should be generated (optional) + :param: nb_assets: the maximum number of pairs to be traded at the same time :return: True if a trade has been created or closed, False otherwise """ state_changed = False try: # Refresh whitelist based on wallet maintenance - refresh_whitelist( + sanitized_list = refresh_whitelist( gen_pair_whitelist( - _CONF['stake_currency'], - topn=dynamic_whitelist - ) if dynamic_whitelist else None + _CONF['stake_currency'] + ) if nb_assets else _CONF['exchange']['pair_whitelist'] ) + + # Keep only the subsets of pairs wanted (up to nb_assets) + final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list + _CONF['exchange']['pair_whitelist'] = final_list + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() if len(trades) < _CONF['max_open_trades']: @@ -295,11 +295,10 @@ def init(config: dict, db_url: Optional[str] = None) -> None: @cached(TTLCache(maxsize=1, ttl=1800)) -def gen_pair_whitelist(base_currency: str, topn: int = 20, key: str = 'BaseVolume') -> List[str]: +def gen_pair_whitelist(base_currency: str, key: str = 'BaseVolume') -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str - :param topn: maximum number of returned results, must be greater than 0 :param key: sort key (defaults to 'BaseVolume') :return: List of pairs """ @@ -309,10 +308,7 @@ def gen_pair_whitelist(base_currency: str, topn: int = 20, key: str = 'BaseVolum reverse=True ) - if topn <= 0: - topn = 20 - - return [s['MarketName'].replace('-', '_') for s in summaries[:topn]] + return [s['MarketName'].replace('-', '_') for s in summaries] def cleanup() -> None: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 57e7c6735..553cd0253 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -268,6 +268,14 @@ CONF_SCHEMA = { 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' }, 'uniqueItems': True + }, + 'pair_blacklist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' + }, + 'uniqueItems': True } }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 3cbc9cfa0..0067eb302 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -41,12 +41,10 @@ def test_refresh_whitelist(mocker): mocker.patch.dict('freqtrade.main._CONF', conf) mocker.patch.multiple('freqtrade.main.exchange', get_wallet_health=get_health) - # no argument: use the whitelist provided by config - refresh_whitelist() + refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist']) whitelist = ['BTC_ETH', 'BTC_TKN'] - pairslist = conf['exchange']['pair_whitelist'] # Ensure all except those in whitelist are removed - assert set(whitelist) == set(pairslist) + assert set(whitelist) == set(refreshedwhitelist) def test_refresh_whitelist_dynamic(mocker): @@ -56,9 +54,8 @@ def test_refresh_whitelist_dynamic(mocker): get_wallet_health=get_health) # argument: use the whitelist dynamically by exchange-volume whitelist = ['BTC_ETH', 'BTC_TKN'] - refresh_whitelist(whitelist) - pairslist = conf['exchange']['pair_whitelist'] - assert set(whitelist) == set(pairslist) + refreshedwhitelist = refresh_whitelist(whitelist) + assert set(whitelist) == set(refreshedwhitelist) def test_refresh_whitelist_dynamic_empty(mocker): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 3d6335572..605ac084b 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -180,6 +180,23 @@ def test_create_trade_no_pairs(default_conf, ticker, mocker): create_trade(default_conf['stake_amount']) +def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy')) + + with pytest.raises(DependencyException, match=r'.*No pair in whitelist.*'): + conf = copy.deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ["BTC_ETH"] + conf['exchange']['pair_blacklist'] = ["BTC_ETH"] + mocker.patch.dict('freqtrade.main._CONF', conf) + create_trade(default_conf['stake_amount']) + + def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) From 49453310931276767c87a41919f18679d05b49ec Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Sat, 30 Dec 2017 15:43:22 +0100 Subject: [PATCH 57/64] Fixing the positional parameter naming + unit tests updated --- freqtrade/main.py | 2 +- freqtrade/tests/test_misc.py | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 18bc04e6f..4a88806c7 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -379,7 +379,7 @@ def main() -> None: throttle( _process, min_secs=_CONF['internals'].get('process_throttle_secs', 10), - dynamic_whitelist=args.dynamic_whitelist, + nb_assets=args.dynamic_whitelist, ) old_state = new_state except KeyboardInterrupt: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 962c833e4..cc75fa01c 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -16,15 +16,33 @@ def test_throttle(): return 42 start = time.time() - result = throttle(func, 0.1) + result = throttle(func, min_secs = 0.1) end = time.time() assert result == 42 assert end - start > 0.1 - result = throttle(func, -1) + result = throttle(func, min_secs = -1) assert result == 42 +def test_throttle_with_assets(): + + def func(nb_assets = -1): + return nb_assets + + start = time.time() + result = throttle(func, min_secs = 0.1, nb_assets = 666) + end = time.time() + + assert result == 666 + + start = time.time() + result = throttle(func, min_secs = 0.1) + end = time.time() + + assert result == -1 + + def test_parse_args_defaults(): args = parse_args([]) From 68f81b2abbe0021603a7b22b736dd8f46dd71e15 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Sat, 30 Dec 2017 15:55:49 +0100 Subject: [PATCH 58/64] autopep8 is going to be my new friend --- freqtrade/tests/test_misc.py | 40 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index cc75fa01c..cd529039a 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -16,34 +16,28 @@ def test_throttle(): return 42 start = time.time() - result = throttle(func, min_secs = 0.1) + result = throttle(func, min_secs=0.1) end = time.time() assert result == 42 assert end - start > 0.1 - result = throttle(func, min_secs = -1) + result = throttle(func, min_secs=-1) assert result == 42 + def test_throttle_with_assets(): - def func(nb_assets = -1): + def func(nb_assets=-1): return nb_assets - start = time.time() - result = throttle(func, min_secs = 0.1, nb_assets = 666) - end = time.time() - + result = throttle(func, min_secs=0.1, nb_assets=666) assert result == 666 - start = time.time() - result = throttle(func, min_secs = 0.1) - end = time.time() - + result = throttle(func, min_secs=0.1) assert result == -1 - def test_parse_args_defaults(): args = parse_args([]) assert args is not None @@ -91,7 +85,8 @@ def test_parse_args_dynamic_whitelist_invalid_values(): def test_parse_args_backtesting(mocker): - backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock()) + backtesting_mock = mocker.patch( + 'freqtrade.optimize.backtesting.start', MagicMock()) args = parse_args(['backtesting']) assert args is None assert backtesting_mock.call_count == 1 @@ -114,7 +109,8 @@ def test_parse_args_backtesting_invalid(): def test_parse_args_backtesting_custom(mocker): - backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock()) + backtesting_mock = mocker.patch( + 'freqtrade.optimize.backtesting.start', MagicMock()) args = parse_args([ '-c', 'test_conf.json', 'backtesting', @@ -135,7 +131,8 @@ def test_parse_args_backtesting_custom(mocker): def test_parse_args_hyperopt(mocker): - hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock()) + hyperopt_mock = mocker.patch( + 'freqtrade.optimize.hyperopt.start', MagicMock()) args = parse_args(['hyperopt']) assert args is None assert hyperopt_mock.call_count == 1 @@ -148,7 +145,8 @@ def test_parse_args_hyperopt(mocker): def test_parse_args_hyperopt_custom(mocker): - hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock()) + hyperopt_mock = mocker.patch( + 'freqtrade.optimize.hyperopt.start', MagicMock()) args = parse_args(['-c', 'test_conf.json', 'hyperopt', '--epochs', '20']) assert args is None assert hyperopt_mock.call_count == 1 @@ -173,7 +171,10 @@ def test_load_config(default_conf, mocker): def test_load_config_invalid_pair(default_conf, mocker): conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'].append('BTC-ETH') - mocker.patch('freqtrade.misc.open', mocker.mock_open(read_data=json.dumps(conf))) + mocker.patch( + 'freqtrade.misc.open', + mocker.mock_open( + read_data=json.dumps(conf))) with pytest.raises(ValidationError, match=r'.*does not match.*'): load_config('somefile') @@ -181,6 +182,9 @@ def test_load_config_invalid_pair(default_conf, mocker): def test_load_config_missing_attributes(default_conf, mocker): conf = deepcopy(default_conf) conf.pop('exchange') - mocker.patch('freqtrade.misc.open', mocker.mock_open(read_data=json.dumps(conf))) + mocker.patch( + 'freqtrade.misc.open', + mocker.mock_open( + read_data=json.dumps(conf))) with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): load_config('somefile') From 1f635d379311611c115671e3c0603766d847c2b1 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 31 Dec 2017 01:14:17 -0800 Subject: [PATCH 59/64] Add pair_blacklist in config.example --- config.json.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.json.example b/config.json.example index f94e423eb..ede081e58 100644 --- a/config.json.example +++ b/config.json.example @@ -29,6 +29,9 @@ "BTC_POWR", "BTC_ADA", "BTC_XMR" + ], + "pair_blacklist": [ + "BTC_DOGE" ] }, "experimental": { From cdfb18e9b4f81eef05f5dd29e558d73702ca51b6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 31 Dec 2017 14:21:50 +0100 Subject: [PATCH 60/64] Update pandas from 0.21.1 to 0.22.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3424c53d7..ace4647e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==2.0.1 requests==2.18.4 urllib3==1.22 wrapt==1.10.11 -pandas==0.21.1 +pandas==0.22.0 scikit-learn==0.19.1 scipy==1.0.0 jsonschema==2.6.0 From de68209f3bf8b3b1f0ad0836ec8c0205e987d672 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Mon, 1 Jan 2018 20:32:58 +0200 Subject: [PATCH 61/64] Revert "Make get_signals async. This should speed up create_trade calls by at least 10x. (#223)" (#275) This reverts commit 67686583007e0a8260d84f525e3a76eef665dcfd. See details in #PR266 --- freqtrade/analyze.py | 3 --- freqtrade/exchange/__init__.py | 4 +--- freqtrade/main.py | 15 ++------------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 887fc0282..d586077db 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -144,9 +144,6 @@ def get_signal(pair: str, signal: SignalType) -> bool: except ValueError as ex: logger.warning('Unable to analyze ticker for pair %s: %s', pair, str(ex)) return False - except Exception: - logger.exception('Unexpected error when analyzing ticker for pair %s.', pair) - return False if dataframe.empty: return False diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8e87cfc8b..7b5c0c753 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -3,7 +3,6 @@ import enum import logging from random import randint -from threading import RLock from typing import List, Dict, Any, Optional import arrow @@ -15,7 +14,6 @@ from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.interface import Exchange logger = logging.getLogger(__name__) -lock = RLock() # Current selected exchange _API: Exchange = None @@ -140,7 +138,7 @@ def get_ticker(pair: str) -> dict: return _API.get_ticker(pair) -@cached(TTLCache(maxsize=100, ttl=30), lock=lock) +@cached(TTLCache(maxsize=100, ttl=30)) def get_ticker_history(pair: str, tick_interval: Optional[int] = 5) -> List[Dict]: return _API.get_ticker_history(pair, tick_interval) diff --git a/freqtrade/main.py b/freqtrade/main.py index afbb84ec4..36dc97aac 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -import asyncio import copy import json import logging import sys import time import traceback -from concurrent.futures import ThreadPoolExecutor from datetime import datetime from typing import Dict, Optional, List @@ -243,17 +241,8 @@ def create_trade(stake_amount: float) -> bool: raise DependencyException('No pair in whitelist') # Pick pair based on StochRSI buy signals - with ThreadPoolExecutor() as pool: - awaitable_signals = [ - asyncio.wrap_future(pool.submit(get_signal, pair, SignalType.BUY)) - for pair in whitelist - ] - - loop = asyncio.get_event_loop() - signals = loop.run_until_complete(asyncio.gather(*awaitable_signals)) - - for idx, _pair in enumerate(whitelist): - if signals[idx]: + for _pair in whitelist: + if get_signal(_pair, SignalType.BUY): pair = _pair break else: From 0e0d61319170c737233d3df14dd460ce3af25f2a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Mon, 1 Jan 2018 20:18:38 +0100 Subject: [PATCH 62/64] Removing tilde and change profit to loss when negative profit is made --- freqtrade/main.py | 6 ++++-- freqtrade/tests/rpc/test_rpc_telegram.py | 6 +++--- freqtrade/tests/test_main.py | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 36dc97aac..8b6c3713a 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -137,8 +137,9 @@ def execute_sell(trade: Trade, limit: float) -> None: _CONF['stake_currency'], _CONF['fiat_display_currency'] ) - message += '` (profit: ~{profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ + message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ '` / {profit_fiat:.3f} {fiat})`'.format( + gain="profit" if fmt_exp_profit > 0 else "loss", profit_percent=fmt_exp_profit, profit_coin=profit_trade, coin=_CONF['stake_currency'], @@ -148,7 +149,8 @@ def execute_sell(trade: Trade, limit: float) -> None: # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well else: - message += '` (profit: ~{profit_percent:.2f}%, {profit_coin:.8f})`'.format( + message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( + gain="profit" if fmt_exp_profit > 0 else "loss", profit_percent=fmt_exp_profit, profit_coin=profit_trade ) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 5a5765ba2..204774c49 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -240,7 +240,7 @@ def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker): assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: ~6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] @@ -277,7 +277,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, m assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: ~-5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] @@ -332,7 +332,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, mocker): assert rpc_mock.call_count == 4 for args in rpc_mock.call_args_list: assert '0.00001098' in args[0][0] - assert 'profit: ~-0.59%, -0.00000591 BTC' in args[0][0] + assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0] assert '-0.089 USD' in args[0][0] diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 73a00d61b..2a2b1e514 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -362,7 +362,7 @@ def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker): assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: ~6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] @@ -399,7 +399,7 @@ def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker): assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: ~-5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] @@ -430,5 +430,5 @@ def test_execute_sell_without_conf(default_conf, ticker, ticker_sell_up, mocker) assert rpc_mock.call_count == 2 assert 'Selling [BTC/ETH]' in rpc_mock.call_args_list[-1][0][0] assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert '(profit: ~6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] + assert '(profit: 6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] assert 'USD' not in rpc_mock.call_args_list[-1][0][0] From 7a2e9ef5356fc055efdcf73e4021c7adfa22844f Mon Sep 17 00:00:00 2001 From: jblestang Date: Mon, 1 Jan 2018 23:21:43 +0100 Subject: [PATCH 63/64] Add fiat display in sell msg (#271) * Display amount (fiat currency) in the sell message * Display also base currency * Adding more info in Buy Message, the stake amount, and the amount using FIAT Converter * fix display style and width * Fixing flake8 --- freqtrade/main.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 8b6c3713a..ff5d4b5aa 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -255,12 +255,21 @@ def create_trade(stake_amount: float) -> bool: amount = stake_amount / buy_limit order_id = exchange.buy(pair, buy_limit, amount) + + fiat_converter = CryptoToFiatConverter() + stake_amount_fiat = fiat_converter.convert_amount( + stake_amount, + _CONF['stake_currency'], + _CONF['fiat_display_currency'] + ) + # Create trade entity and return - rpc.send_msg('*{}:* Buying [{}]({}) with limit `{:.8f}`'.format( + rpc.send_msg('*{}:* Buying [{}]({}) with limit `{:.8f} ({:.6f} {}, {:.3f} {})` '.format( exchange.get_name().upper(), pair.replace('_', '/'), exchange.get_pair_detail_url(pair), - buy_limit + buy_limit, stake_amount, _CONF['stake_currency'], + stake_amount_fiat, _CONF['fiat_display_currency'] )) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL trade = Trade( From 50be2fabbf379710175ee974ccb201add5bf61f7 Mon Sep 17 00:00:00 2001 From: Stephen Date: Tue, 2 Jan 2018 15:04:41 +1100 Subject: [PATCH 64/64] Fixed pytest typo --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e8b0684e9..21dcda44a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ pytest freqtrade/tests/test_.py **Test only one method from one file** ```bash -pytest freqtrade/tests/test_.py:test_ +pytest freqtrade/tests/test_.py::test_ ``` ## Test if your code is PEP8 compliant **Install packages** (If not already installed)