From 5623ea3ac65ff832f5b680abf00592c7e0ba77fd Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 9 Jun 2018 21:44:20 +0200 Subject: [PATCH 01/59] Add forcesell at end of backtest period --- freqtrade/optimize/backtesting.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 028a4f521..2b1bb98a0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -130,6 +130,21 @@ class Backtesting(object): (sell_row.date - buy_row.date).seconds // 60 ), \ sell_row.date + if partial_ticker: + # no sell condition found - trade stil open at end of backtest period + sell_row = partial_ticker[-1] + logger.info('Force_selling still open trade %s with %s perc - %s', pair, + trade.calc_profit_percent(rate=sell_row.close), + trade.calc_profit(rate=sell_row.close)) + return \ + sell_row, \ + ( + pair, + trade.calc_profit_percent(rate=sell_row.close), + trade.calc_profit(rate=sell_row.close), + (sell_row.date - buy_row.date).seconds // 60 + ), \ + sell_row.date return None def backtest(self, args: Dict) -> DataFrame: @@ -170,6 +185,7 @@ class Backtesting(object): ticker_data.drop(ticker_data.head(1).index, inplace=True) + # TODO: why convert from Pandas to list?? ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None @@ -202,6 +218,11 @@ class Backtesting(object): row.date.strftime('%s'), row2.date.strftime('%s'), index, trade_entry[3])) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date + # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: From 24a875ed465442100befb6f07d626e7caeb37a25 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 9 Jun 2018 21:44:57 +0200 Subject: [PATCH 02/59] remove experimental parameters - they are read by analyze.py anyway --- freqtrade/optimize/backtesting.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2b1bb98a0..56a9ca9bc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -277,16 +277,12 @@ class Backtesting(object): ) # Execute backtest and print results - sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False) - use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False) results = self.backtest( { 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, 'realistic': self.config.get('realistic_simulation', False), - 'sell_profit_only': sell_profit_only, - 'use_sell_signal': use_sell_signal, 'record': self.config.get('export'), 'recordfn': self.config.get('exportfilename'), } From 3094acc7fb33559f7212e313677710131af9ec9f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 08:58:28 +0200 Subject: [PATCH 03/59] update comment --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 56a9ca9bc..d6748bd76 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -185,7 +185,8 @@ class Backtesting(object): ticker_data.drop(ticker_data.head(1).index, inplace=True) - # TODO: why convert from Pandas to list?? + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None From c1b2e06edad52dc91602088c67c4a97b05780b16 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 09:07:04 +0200 Subject: [PATCH 04/59] simplify return from _get_sell_trade_entry --- freqtrade/optimize/backtesting.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d6748bd76..acb419ef5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -128,8 +128,7 @@ class Backtesting(object): trade.calc_profit_percent(rate=sell_row.close), trade.calc_profit(rate=sell_row.close), (sell_row.date - buy_row.date).seconds // 60 - ), \ - sell_row.date + ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period sell_row = partial_ticker[-1] @@ -143,8 +142,7 @@ class Backtesting(object): trade.calc_profit_percent(rate=sell_row.close), trade.calc_profit(rate=sell_row.close), (sell_row.date - buy_row.date).seconds // 60 - ), \ - sell_row.date + ) return None def backtest(self, args: Dict) -> DataFrame: @@ -208,8 +206,8 @@ class Backtesting(object): trade_count_lock, args) if ret: - row2, trade_entry, next_date = ret - lock_pair_until = next_date + row2, trade_entry = ret + lock_pair_until = row2.date trades.append(trade_entry) if record: # Note, need to be json.dump friendly From 9c57d3aa8b98c95fbd31fb80e2323fa80bb537ac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:15:25 +0200 Subject: [PATCH 05/59] add BacktestresultTuple --- freqtrade/optimize/backtesting.py | 88 +++++++++++++++++-------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index acb419ef5..574b7b283 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,7 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace -from typing import Dict, Tuple, Any, List, Optional +from typing import Dict, Tuple, Any, List, Optional, NamedTuple import arrow from pandas import DataFrame @@ -23,6 +23,18 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) +class BacktestResult(NamedTuple): + """ + NamedTuple Defining BacktestResults inputs. + """ + pair: str + profit_percent: float + profit_abs: float + open_time: float + close_time: float + trade_duration: float + + class Backtesting(object): """ Backtesting class, this class contains all the logic to run a backtest @@ -73,15 +85,15 @@ class Backtesting(object): headers = ['pair', 'buy count', 'avg profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: - result = results[results.currency == pair] + result = results[results.pair == pair] tabular_data.append([ pair, len(result.index), result.profit_percent.mean() * 100.0, - result.profit_BTC.sum(), - result.duration.mean(), - len(result[result.profit_BTC > 0]), - len(result[result.profit_BTC < 0]) + result.profit_abs.sum(), + result.trade_duration.mean(), + len(result[result.profit_abs > 0]), + len(result[result.profit_abs < 0]) ]) # Append Total @@ -89,16 +101,16 @@ class Backtesting(object): 'TOTAL', len(results.index), results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.duration.mean(), - len(results[results.profit_BTC > 0]), - len(results[results.profit_BTC < 0]) + results.profit_abs.sum(), + results.trade_duration.mean(), + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, - partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[Tuple]: + partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]: stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) @@ -121,28 +133,27 @@ class Backtesting(object): buy_signal = sell_row.buy if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal, sell_row.sell): - return \ - sell_row, \ - ( - pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close), - (sell_row.date - buy_row.date).seconds // 60 - ) + + return BacktestResult(pair=pair, + profit_percent=trade.calc_profit_percent(rate=sell_row.close), + profit_abs=trade.calc_profit(rate=sell_row.close), + open_time=buy_row.date, + close_time=sell_row.date, + trade_duration=(sell_row.date - buy_row.date).seconds // 60 + ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period sell_row = partial_ticker[-1] - logger.info('Force_selling still open trade %s with %s perc - %s', pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close)) - return \ - sell_row, \ - ( - pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close), - (sell_row.date - buy_row.date).seconds // 60 - ) + btr = BacktestResult(pair=pair, + profit_percent=trade.calc_profit_percent(rate=sell_row.close), + profit_abs=trade.calc_profit(rate=sell_row.close), + open_time=buy_row.date, + close_time=sell_row.date, + trade_duration=(sell_row.date - buy_row.date).seconds // 60 + ) + logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, + btr.profit_percent, btr.profit_abs) + return btr return None def backtest(self, args: Dict) -> DataFrame: @@ -202,20 +213,19 @@ class Backtesting(object): trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - ret = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - if ret: - row2, trade_entry = ret - lock_pair_until = row2.date + if trade_entry: + lock_pair_until = trade_entry.close_time trades.append(trade_entry) if record: # Note, need to be json.dump friendly # record a tuple of pair, current_profit_percent, # entry-date, duration - records.append((pair, trade_entry[1], - row.date.strftime('%s'), - row2.date.strftime('%s'), + records.append((pair, trade_entry.profit_percent, + trade_entry.open_time.strftime('%s'), + trade_entry.close_time.strftime('%s'), index, trade_entry[3])) else: # Set lock_pair_until to end of testing period if trade could not be closed @@ -228,7 +238,7 @@ class Backtesting(object): logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] - return DataFrame.from_records(trades, columns=labels) + return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: """ From 7b5a2946e5b8636a4fd6d181b976951b03ede96f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:19:32 +0200 Subject: [PATCH 06/59] adjust for forcesell backtesting --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 33d9703de..72de5eb9c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -538,7 +538,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): backtesting.populate_buy_trend = _trend_alternate # Override backtesting.populate_sell_trend = _trend_alternate # Override results = backtesting.backtest(backtest_conf) - assert len(results) == 3 + assert len(results) == 4 def test_backtest_record(default_conf, fee, mocker): From c9476fade8180bb57b3fb7f123a25badd174a48d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:20:41 +0200 Subject: [PATCH 07/59] adjust tests for forcesell --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 72de5eb9c..ad328edc2 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -478,7 +478,7 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) - tests = [['raise', 17], ['lower', 0], ['sine', 16]] + tests = [['raise', 18], ['lower', 0], ['sine', 16]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres, mocker) From 17c0ceec04b8cb9f826a80bee1a321bef77874b1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:22:24 +0200 Subject: [PATCH 08/59] adjust tests for backtestresult type --- freqtrade/tests/optimize/test_backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index ad328edc2..d5ac6e01e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -353,10 +353,10 @@ def test_generate_text_table(default_conf, mocker): results = pd.DataFrame( { - 'currency': ['ETH/BTC', 'ETH/BTC'], + 'pair': ['ETH/BTC', 'ETH/BTC'], 'profit_percent': [0.1, 0.2], - 'profit_BTC': [0.2, 0.4], - 'duration': [10, 30], + 'profit_abs': [0.2, 0.4], + 'trade_duration': [10, 30], 'profit': [2, 0], 'loss': [0, 0] } From 322a528c12aacc4712c5d0685d1d582dd3e6747a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:25:16 +0200 Subject: [PATCH 09/59] fix bug with backtestResult --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 574b7b283..cd020a6db 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -226,7 +226,7 @@ class Backtesting(object): records.append((pair, trade_entry.profit_percent, trade_entry.open_time.strftime('%s'), trade_entry.close_time.strftime('%s'), - index, trade_entry[3])) + index, trade_entry.trade_duration)) else: # Set lock_pair_until to end of testing period if trade could not be closed # This happens only if the buy-signal was with the last candle From aff1ede46bf88817e402d090a289bb39b699e291 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:25:52 +0200 Subject: [PATCH 10/59] Fix last backtesting test --- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index d5ac6e01e..7c617c38f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -560,12 +560,12 @@ def test_backtest_record(default_conf, fee, mocker): backtesting.populate_buy_trend = _trend_alternate # Override backtesting.populate_sell_trend = _trend_alternate # Override results = backtesting.backtest(backtest_conf) - assert len(results) == 3 + assert len(results) == 4 # Assert file_dump_json was only called once assert names == ['backtest-result.json'] records = records[0] # Ensure records are of correct type - assert len(records) == 3 + assert len(records) == 4 # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None From 31025216f94f58f217841f1215a7869a684bbd2c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:32:07 +0200 Subject: [PATCH 11/59] fix type of open/close timestmap --- freqtrade/optimize/backtesting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cd020a6db..68480c997 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,6 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace +from datetime import datetime from typing import Dict, Tuple, Any, List, Optional, NamedTuple import arrow @@ -30,8 +31,8 @@ class BacktestResult(NamedTuple): pair: str profit_percent: float profit_abs: float - open_time: float - close_time: float + open_time: datetime + close_time: datetime trade_duration: float From b81588307f0e14d6e6ec82073e86f4e7f98cbaf7 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:37:53 +0200 Subject: [PATCH 12/59] Add "open_at_end" parameter --- freqtrade/optimize/backtesting.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 68480c997..f87340829 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -34,6 +34,7 @@ class BacktestResult(NamedTuple): open_time: datetime close_time: datetime trade_duration: float + open_at_end: bool class Backtesting(object): @@ -140,7 +141,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.close), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60 + trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_at_end=False ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -150,7 +152,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.close), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60 + trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_at_end=True ) logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) From 1cd7ac55a8d742f7c7e2278685cdc412205eacd0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:45:16 +0200 Subject: [PATCH 13/59] Added "left open trades" report --- freqtrade/optimize/backtesting.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f87340829..cc20e9789 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -311,6 +311,17 @@ class Backtesting(object): ) ) + logger.info( + '\n==================================== ' + 'LEFT OPEN TRADES REPORT' + ' ====================================\n' + '%s', + self._generate_text_table( + data, + results.loc[results.open_at_end == True] + ) + ) + def setup_configuration(args: Namespace) -> Dict[str, Any]: """ From 27ee8f73604b52da8e7a5865d52949afd9b99a7b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:55:48 +0200 Subject: [PATCH 14/59] make flake happy --- freqtrade/optimize/backtesting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cc20e9789..e526a6ec4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -241,7 +241,6 @@ class Backtesting(object): if record and record.find('trades') >= 0: logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: @@ -318,7 +317,7 @@ class Backtesting(object): '%s', self._generate_text_table( data, - results.loc[results.open_at_end == True] + results.loc[results.open_at_end] ) ) From 4710210cffe52fcac5406d1d1840732253c1073f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:56:10 +0200 Subject: [PATCH 15/59] fix hyperopt to use new backtesting result tuple --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 878acc2dc..e952458a7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -451,7 +451,7 @@ class Hyperopt(Backtesting): total_profit = results.profit_percent.sum() trade_count = len(results.index) - trade_duration = results.duration.mean() + trade_duration = results.trade_duration.mean() if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') @@ -488,10 +488,10 @@ class Hyperopt(Backtesting): 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), + results.profit_abs.sum(), self.config['stake_currency'], results.profit_percent.sum(), - results.duration.mean(), + results.trade_duration.mean(), ) def start(self) -> None: From 9cc087c788bd56fc230d8bafa6993886d7ceb514 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 13:56:23 +0200 Subject: [PATCH 16/59] update hyperopt tests to support new structure --- freqtrade/tests/optimize/test_hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 3edfe4393..62fd17b82 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -400,7 +400,7 @@ def test_format_results(init_hyperopt): ('LTC/BTC', 1, 1, 123), ('XPR/BTC', -1, -2, -246) ] - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration'] df = pd.DataFrame.from_records(trades, columns=labels) result = _HYPEROPT.format_results(df) @@ -530,7 +530,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: trades = [ ('POWR/BTC', 0.023117, 0.000233, 100) ] - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration'] backtest_result = pd.DataFrame.from_records(trades, columns=labels) mocker.patch( From 12e455cbf5616462d0ca198eb2122e13ba17add8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 10 Jun 2018 20:52:42 +0200 Subject: [PATCH 17/59] add buy/sell index to backtest result --- freqtrade/optimize/backtesting.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e526a6ec4..ffd6e2768 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,6 +33,8 @@ class BacktestResult(NamedTuple): profit_abs: float open_time: datetime close_time: datetime + open_index: int + close_index: int trade_duration: float open_at_end: bool @@ -142,6 +144,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_index=buy_row.index, + close_index=sell_row.index, open_at_end=False ) if partial_ticker: @@ -153,6 +157,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, + open_index=buy_row.index, + close_index=sell_row.index, open_at_end=True ) logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, From 335d1fbbbcff5b09538f47076294914a37166412 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 11 Jun 2018 19:50:43 +0200 Subject: [PATCH 18/59] Check if no backtest data is found and fail gracefully --- freqtrade/optimize/backtesting.py | 3 ++ freqtrade/tests/optimize/test_backtesting.py | 40 ++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 028a4f521..713f23ecc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -237,6 +237,9 @@ class Backtesting(object): timerange=timerange ) + if not data: + logger.critical("No data found. Terminating.") + return # Ignore max_open_trades in backtesting, except realistic flag was passed if self.config.get('realistic_simulation', False): max_open_trades = self.config['max_open_trades'] diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 33d9703de..41a00938c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -416,6 +416,46 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: assert log_has(line, caplog.record_tuples) +def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: + """ + Test Backtesting.start() method if no data is found + """ + + def get_timeframe(input1, input2): + return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.get_ticker_history') + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch.multiple( + 'freqtrade.optimize.backtesting.Backtesting', + backtest=MagicMock(), + _generate_text_table=MagicMock(return_value='1'), + get_timeframe=get_timeframe, + ) + + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + conf['ticker_interval'] = 1 + conf['live'] = False + conf['datadir'] = None + conf['export'] = None + conf['timerange'] = '20180101-20180102' + + backtesting = Backtesting(conf) + backtesting.start() + # check the logs, that will contain the backtest result + exists = [ + 'Using local backtesting data (using whitelist in given config) ...', + 'Using stake_currency: BTC ...', + 'Using stake_amount: 0.001 ...', + 'No data found. Terminating.' + ] + for line in exists: + assert log_has(line, caplog.record_tuples) + + def test_backtest(default_conf, fee, mocker) -> None: """ Test Backtesting.backtest() method From a0f735d4f268d60c46236a7e115d874f833c0dbd Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 11 Jun 2018 21:02:24 +0200 Subject: [PATCH 19/59] reduce test-noise --- freqtrade/tests/optimize/test_backtesting.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 41a00938c..8c18cab4e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -437,7 +437,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - conf['ticker_interval'] = 1 + conf['ticker_interval'] = "1m" conf['live'] = False conf['datadir'] = None conf['export'] = None @@ -446,14 +446,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: backtesting = Backtesting(conf) backtesting.start() # check the logs, that will contain the backtest result - exists = [ - 'Using local backtesting data (using whitelist in given config) ...', - 'Using stake_currency: BTC ...', - 'Using stake_amount: 0.001 ...', - 'No data found. Terminating.' - ] - for line in exists: - assert log_has(line, caplog.record_tuples) + + assert log_has('No data found. Terminating.', caplog.record_tuples) def test_backtest(default_conf, fee, mocker) -> None: From 40746c3fcb93eb01f4ce69688353bf9cd780fec0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 11 Jun 2018 21:10:57 +0200 Subject: [PATCH 20/59] fix downloadscript crash if a pair is not available --- scripts/download_backtest_data.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 6185edaf7..9aedbecb9 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -30,6 +30,8 @@ if not os.path.isfile(pairs_file): with open(pairs_file) as file: PAIRS = list(set(json.load(file))) +PAIRS.sort() + since_time = None if args.days: since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000 @@ -41,9 +43,15 @@ print(f'About to download pairs: {PAIRS} to {dl_path}') exchange._API = exchange.init_ccxt({'key': '', 'secret': '', 'name': args.exchange}) - +pairs_not_available = [] +# Make sure API markets is initialized +exchange._API.load_markets() for pair in PAIRS: + if pair not in exchange._API.markets: + pairs_not_available.append(pair) + print(f"skipping pair {pair}") + continue for tick_interval in timeframes: print(f'downloading pair {pair}, interval {tick_interval}') @@ -60,3 +68,7 @@ for pair in PAIRS: pair_print = pair.replace('/', '_') filename = f'{pair_print}-{tick_interval}.json' misc.file_dump_json(os.path.join(dl_path, filename), data) + + +if pairs_not_available: + print(f"Pairs [{','.join(pairs_not_available)}] not availble.") From 06b71d713cec7c8df88d5b1c2b9c31fd3b9e0195 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 07:00:58 +0200 Subject: [PATCH 21/59] update issue template to include ccxt version --- .github/ISSUE_TEMPLATE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 2a6d3f18f..94d998310 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,10 +6,12 @@ If it hasn't been reported, please create a new issue. ## Step 2: Describe your environment * Python Version: _____ (`python -V`) + * CCXT version: _____ (`pip freeze | grep ccxt`) * Branch: Master | Develop * Last Commit ID: _____ (`git log --format="%H" -n 1`) - + ## Step 3: Describe the problem: + *Explain the problem you have encountered* ### Steps to reproduce: From aa6e276cf9e4bdb26c71e922361ad8b3e7f84289 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 12 Jun 2018 14:22:06 +0200 Subject: [PATCH 22/59] Update ccxt from 1.14.172 to 1.14.177 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 63e1a3891..2272d47b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.172 +ccxt==1.14.177 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From bfde33c945f4374fc8ca69075d41a2d9c731d6e9 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 21:12:55 +0200 Subject: [PATCH 23/59] Use timestamp() instead of strftime this will avoid a bug shifting epoch time by 1 hour: https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffd6e2768..f7fade102 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -234,8 +234,8 @@ class Backtesting(object): # record a tuple of pair, current_profit_percent, # entry-date, duration records.append((pair, trade_entry.profit_percent, - trade_entry.open_time.strftime('%s'), - trade_entry.close_time.strftime('%s'), + trade_entry.open_time.timestamp(), + trade_entry.close_time.timestamp(), index, trade_entry.trade_duration)) else: # Set lock_pair_until to end of testing period if trade could not be closed From 1f6b9c332b5d3f38bed26dd539ef29d44fae6ae1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 21:38:14 +0200 Subject: [PATCH 24/59] fix default datadir not working in plot-script --- scripts/plot_profit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 803bf71de..012446065 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -121,7 +121,7 @@ def plot_profit(args: Namespace) -> None: logger.info('Filter, keep pairs %s' % pairs) tickers = optimize.load_data( - datadir=args.datadir, + datadir=config.get('datadir'), pairs=pairs, ticker_interval=tick_interval, refresh_pairs=False, From 182f4c603be87097cfd9b8229f6db688171250a7 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 21:43:14 +0200 Subject: [PATCH 25/59] fix plot-script datadir not working --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 122c002a8..ce1a4b819 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -91,7 +91,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data( - datadir=args.datadir, + datadir=_CONF.get("datadir"), pairs=[pair], ticker_interval=tick_interval, refresh_pairs=_CONF.get('refresh_pairs', False), From e3ced7c15e0d1dc0582485147d59ca5122efebfc Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 12 Jun 2018 22:29:30 +0200 Subject: [PATCH 26/59] extract export from backtest function --- freqtrade/optimize/backtesting.py | 47 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f7fade102..1146d6b00 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -112,6 +112,21 @@ class Backtesting(object): ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: + + records = [] + print(results) + for index, trade_entry in results.iterrows(): + pass + records.append((trade_entry.pair, trade_entry.profit_percent, + trade_entry.open_time.timestamp(), + trade_entry.close_time.timestamp(), + trade_entry.open_index - 1, trade_entry.trade_duration)) + + if records: + logger.info('Dumping backtest results to %s', recordfilename) + file_dump_json(recordfilename, records) + def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]: @@ -144,8 +159,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, - open_index=buy_row.index, - close_index=sell_row.index, + open_index=buy_row.Index, + close_index=sell_row.Index, open_at_end=False ) if partial_ticker: @@ -157,8 +172,8 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, - open_index=buy_row.index, - close_index=sell_row.index, + open_index=buy_row.Index, + close_index=sell_row.Index, open_at_end=True ) logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, @@ -179,17 +194,12 @@ class Backtesting(object): processed: a processed dictionary with format {pair, data} max_open_trades: maximum number of concurrent trades (default: 0, disabled) realistic: do we try to simulate realistic trades? (default: True) - sell_profit_only: sell if profit only - use_sell_signal: act on sell-signal :return: DataFrame """ headers = ['date', 'buy', 'open', 'close', 'sell'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) - record = args.get('record', None) - recordfilename = args.get('recordfn', 'backtest-result.json') - records = [] trades = [] trade_count_lock: Dict = {} for pair, pair_data in processed.items(): @@ -229,24 +239,11 @@ class Backtesting(object): if trade_entry: lock_pair_until = trade_entry.close_time trades.append(trade_entry) - if record: - # Note, need to be json.dump friendly - # record a tuple of pair, current_profit_percent, - # entry-date, duration - records.append((pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - index, trade_entry.trade_duration)) else: # Set lock_pair_until to end of testing period if trade could not be closed # This happens only if the buy-signal was with the last candle lock_pair_until = ticker_data.iloc[-1].date - # For now export inside backtest(), maybe change so that backtest() - # returns a tuple like: (dataframe, records, logs, etc) - if record and record.find('trades') >= 0: - logger.info('Dumping backtest results to %s', recordfilename) - file_dump_json(recordfilename, records) return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: @@ -301,10 +298,12 @@ class Backtesting(object): 'processed': preprocessed, 'max_open_trades': max_open_trades, 'realistic': self.config.get('realistic_simulation', False), - 'record': self.config.get('export'), - 'recordfn': self.config.get('exportfilename'), } ) + + if self.config.get('export', False): + self._store_backtest_result(self.config.get('exportfilename'), results) + logger.info( '\n==================================== ' 'BACKTESTING REPORT' From 8d8e6dcffc60cabd6c135d95337401d9abb5b69b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:31:42 +0200 Subject: [PATCH 27/59] Add test for extracted backtest_results test --- freqtrade/tests/optimize/test_backtesting.py | 26 +++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 7c617c38f..e0342851f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -550,16 +550,24 @@ def test_backtest_record(default_conf, fee, mocker): 'freqtrade.optimize.backtesting.file_dump_json', new=lambda n, r: (names.append(n), records.append(r)) ) - backtest_conf = _make_backtest_conf( - mocker, - conf=default_conf, - pair='UNITTEST/BTC', - record="trades" - ) + backtesting = Backtesting(default_conf) - backtesting.populate_buy_trend = _trend_alternate # Override - backtesting.populate_sell_trend = _trend_alternate # Override - results = backtesting.backtest(backtest_conf) + results = pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_index": [1, 119, 153, 185], + "close_index": [118, 151, 184, 199], + "trade_duration": [123, 34, 31, 14]}) + backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 # Assert file_dump_json was only called once assert names == ['backtest-result.json'] From 0f117d480e4f8822c76bbda2e4cd24300b495f50 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:42:24 +0200 Subject: [PATCH 28/59] improve backtesting-tests * assert length of result specifically * add assert for "open_at_end" --- freqtrade/tests/optimize/test_backtesting.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e0342851f..15f1d978e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -435,6 +435,7 @@ def test_backtest(default_conf, fee, mocker) -> None: } ) assert not results.empty + assert len(results) == 2 def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: @@ -457,6 +458,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: } ) assert not results.empty + assert len(results) == 1 def test_processed(default_conf, mocker) -> None: @@ -538,7 +540,10 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): backtesting.populate_buy_trend = _trend_alternate # Override backtesting.populate_sell_trend = _trend_alternate # Override results = backtesting.backtest(backtest_conf) + backtesting._store_backtest_result("test_.json", results) assert len(results) == 4 + # One trade was force-closed at the end + assert len(results.loc[results.open_at_end]) == 1 def test_backtest_record(default_conf, fee, mocker): From 6e68c3b230490e47cb84b27ba1ebb9fd176d32b9 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:52:17 +0200 Subject: [PATCH 29/59] fix backtesting.md formatting --- docs/backtesting.md | 52 +++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 8364d77e4..e46999d4b 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -1,17 +1,19 @@ # Backtesting + This page explains how to validate your strategy performance by using Backtesting. ## Table of Contents + - [Test your strategy with Backtesting](#test-your-strategy-with-backtesting) - [Understand the backtesting result](#understand-the-backtesting-result) ## Test your strategy with Backtesting + Now you have good Buy and Sell strategies, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). - Backtesting will use the crypto-currencies (pair) from your config file and load static tickers located in [/freqtrade/tests/testdata](https://github.com/freqtrade/freqtrade/tree/develop/freqtrade/tests/testdata). @@ -19,70 +21,80 @@ If the 5 min and 1 min ticker for the crypto-currencies to test is not already in the `testdata` folder, backtesting will download them automatically. Testdata files will not be updated until you specify it. -The result of backtesting will confirm you if your bot as more chance to -make a profit than a loss. - +The result of backtesting will confirm you if your bot as more chance to make a profit than a loss. The backtesting is very easy with freqtrade. ### Run a backtesting against the currencies listed in your config file -**With 5 min tickers (Per default)** +#### With 5 min tickers (Per default) + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation ``` -**With 1 min tickers** +#### With 1 min tickers + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --ticker-interval 1m ``` -**Update cached pairs with the latest data** +#### Update cached pairs with the latest data + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-cached ``` -**With live data (do not alter your testdata files)** +#### With live data (do not alter your testdata files) + ```bash python3 ./freqtrade/main.py backtesting --realistic-simulation --live ``` -**Using a different on-disk ticker-data source** +#### Using a different on-disk ticker-data source + ```bash python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101 ``` -**With a (custom) strategy file** +#### With a (custom) strategy file + ```bash python3 ./freqtrade/main.py -s TestStrategy backtesting ``` + Where `-s TestStrategy` refers to the class name within the strategy file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory -**Exporting trades to file** +#### Exporting trades to file + ```bash python3 ./freqtrade/main.py backtesting --export trades ``` -**Exporting trades to file specifying a custom filename** +#### Exporting trades to file specifying a custom filename + ```bash python3 ./freqtrade/main.py backtesting --export trades --export-filename=backtest_teststrategy.json ``` +#### Running backtest with smaller testset -**Running backtest with smaller testset** Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: + ```bash python3 ./freqtrade/main.py backtesting --timerange=-200 ``` -***Advanced use of timerange*** +#### Advanced use of timerange + Doing `--timerange=-200` will get the last 200 timeframes from your inputdata. You can also specify specific dates, or a range span indexed by start and stop. The full timerange specification: + - Use last 123 tickframes of data: `--timerange=-123` - Use first 123 tickframes of data: `--timerange=123-` - Use tickframes from line 123 through 456: `--timerange=123-456` @@ -92,11 +104,12 @@ The full timerange specification: - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` +#### Downloading new set of ticker data -**Downloading new set of ticker data** To download new set of backtesting ticker data, you can use a download script. If you are using Binance for example: + - create a folder `user_data/data/binance` and copy `pairs.json` in that folder. - update the `pairs.json` to contain the currency pairs you are interested in. @@ -119,14 +132,14 @@ This will download ticker data for all the currency pairs you defined in `pairs. - To download ticker data for only 10 days, use `--days 10`. - Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. - -For help about backtesting usage, please refer to -[Backtesting commands](#backtesting-commands). +For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). ## Understand the backtesting result + The most important in the backtesting is to understand the result. A backtesting result will look like that: + ``` ====================== BACKTESTING REPORT ================================ pair buy count avg profit % total profit BTC avg duration @@ -146,6 +159,7 @@ TOTAL 419 -0.41 -0.00348593 52.9 The last line will give you the overall performance of your strategy, here: + ``` TOTAL 419 -0.41 -0.00348593 52.9 ``` @@ -161,6 +175,7 @@ strategy, your sell strategy, and also by the `minimal_roi` and As for an example if your minimal_roi is only `"0": 0.01`. You cannot expect the bot to make more profit than 1% (because it will sell every time a trade will reach 1%). + ```json "minimal_roi": { "0": 0.01 @@ -173,6 +188,7 @@ profit. Hence, keep in mind that your performance is a mix of your strategies, your configuration, and the crypto-currency you have set up. ## Next step + Great, your strategy is profitable. What if the bot can give your the optimal parameters to use for your strategy? Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) From 6357812743defe2cd3208d00dc37b4415409a841 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 06:57:49 +0200 Subject: [PATCH 30/59] fix backtest report able --- freqtrade/optimize/backtesting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1146d6b00..5df4fb28a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -305,9 +305,9 @@ class Backtesting(object): self._store_backtest_result(self.config.get('exportfilename'), results) logger.info( - '\n==================================== ' + '\n======================================== ' 'BACKTESTING REPORT' - ' ====================================\n' + ' =========================================\n' '%s', self._generate_text_table( data, @@ -316,9 +316,9 @@ class Backtesting(object): ) logger.info( - '\n==================================== ' + '\n====================================== ' 'LEFT OPEN TRADES REPORT' - ' ====================================\n' + ' ======================================\n' '%s', self._generate_text_table( data, From e22da45474aee9487d2e2a38556bcac987bd88b2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 07:00:05 +0200 Subject: [PATCH 31/59] update documentation with forcesell at the end of the backtest period --- docs/backtesting.md | 49 ++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index e46999d4b..127b4ee20 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -141,22 +141,43 @@ The most important in the backtesting is to understand the result. A backtesting result will look like that: ``` -====================== BACKTESTING REPORT ================================ -pair buy count avg profit % total profit BTC avg duration --------- ----------- -------------- ------------------ -------------- -ETH/BTC 56 -0.67 -0.00075455 62.3 -LTC/BTC 38 -0.48 -0.00036315 57.9 -ETC/BTC 42 -1.15 -0.00096469 67.0 -DASH/BTC 72 -0.62 -0.00089368 39.9 -ZEC/BTC 45 -0.46 -0.00041387 63.2 -XLM/BTC 24 -0.88 -0.00041846 47.7 -NXT/BTC 24 0.68 0.00031833 40.2 -POWR/BTC 35 0.98 0.00064887 45.3 -ADA/BTC 43 -0.39 -0.00032292 55.0 -XMR/BTC 40 -0.40 -0.00032181 47.4 -TOTAL 419 -0.41 -0.00348593 52.9 +======================================== BACKTESTING REPORT ========================================= +| pair | buy count | avg profit % | total profit BTC | avg duration | profit | loss | +|:---------|------------:|---------------:|-------------------:|---------------:|---------:|-------:| +| ETH/BTC | 44 | 0.18 | 0.00159118 | 50.9 | 44 | 0 | +| LTC/BTC | 27 | 0.10 | 0.00051931 | 103.1 | 26 | 1 | +| ETC/BTC | 24 | 0.05 | 0.00022434 | 166.0 | 22 | 2 | +| DASH/BTC | 29 | 0.18 | 0.00103223 | 192.2 | 29 | 0 | +| ZEC/BTC | 65 | -0.02 | -0.00020621 | 202.7 | 62 | 3 | +| XLM/BTC | 35 | 0.02 | 0.00012877 | 242.4 | 32 | 3 | +| BCH/BTC | 12 | 0.62 | 0.00149284 | 50.0 | 12 | 0 | +| POWR/BTC | 21 | 0.26 | 0.00108215 | 134.8 | 21 | 0 | +| ADA/BTC | 54 | -0.19 | -0.00205202 | 191.3 | 47 | 7 | +| XMR/BTC | 24 | -0.43 | -0.00206013 | 120.6 | 20 | 4 | +| TOTAL | 335 | 0.03 | 0.00175246 | 157.9 | 315 | 20 | +2018-06-13 06:57:27,347 - freqtrade.optimize.backtesting - INFO - +====================================== LEFT OPEN TRADES REPORT ====================================== +| pair | buy count | avg profit % | total profit BTC | avg duration | profit | loss | +|:---------|------------:|---------------:|-------------------:|---------------:|---------:|-------:| +| ETH/BTC | 3 | 0.16 | 0.00009619 | 25.0 | 3 | 0 | +| LTC/BTC | 1 | -1.00 | -0.00020118 | 1085.0 | 0 | 1 | +| ETC/BTC | 2 | -1.80 | -0.00071933 | 1092.5 | 0 | 2 | +| DASH/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| ZEC/BTC | 3 | -4.27 | -0.00256826 | 1301.7 | 0 | 3 | +| XLM/BTC | 3 | -1.11 | -0.00066744 | 965.0 | 0 | 3 | +| BCH/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| POWR/BTC | 0 | nan | 0.00000000 | nan | 0 | 0 | +| ADA/BTC | 7 | -3.58 | -0.00503604 | 850.0 | 0 | 7 | +| XMR/BTC | 4 | -3.79 | -0.00303456 | 291.2 | 0 | 4 | +| TOTAL | 23 | -2.63 | -0.01213062 | 750.4 | 3 | 20 | + ``` +The 1st table will contain all trades the bot made. + +The 2nd table will contain all trades the bot had to `forcesell` at the end of the backtest period to prsent a full picture. +These trades are also included in the first table, but are extracted separately for clarity. + The last line will give you the overall performance of your strategy, here: From cddb062db52252a50b519890f882def8e0aada54 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Jun 2018 03:49:09 +0200 Subject: [PATCH 32/59] save rpc instances only in registered_modules, add some abstract methods --- freqtrade/rpc/rpc.py | 17 +++++++- freqtrade/rpc/rpc_manager.py | 48 ++++++++-------------- freqtrade/rpc/telegram.py | 52 ++++++++++++++---------- freqtrade/tests/rpc/test_rpc_manager.py | 38 +++++------------ freqtrade/tests/rpc/test_rpc_telegram.py | 42 +++++++++---------- freqtrade/tests/test_freqtradebot.py | 2 +- 6 files changed, 95 insertions(+), 104 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 33cfc3e8f..5ac78f2cb 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -2,21 +2,21 @@ This module contains class to define a RPC communications """ import logging +from abc import abstractmethod from datetime import datetime, timedelta, date from decimal import Decimal from typing import Dict, Tuple, Any import arrow import sqlalchemy as sql -from pandas import DataFrame from numpy import mean, nan_to_num +from pandas import DataFrame from freqtrade import exchange from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State - logger = logging.getLogger(__name__) @@ -32,6 +32,19 @@ class RPC(object): """ self.freqtrade = freqtrade + @abstractmethod + def cleanup(self) -> str: + """ Cleanup pending module resources """ + + @property + @abstractmethod + def name(self) -> None: + """ Returns the lowercase name of this module """ + + @abstractmethod + def send_msg(self, msg: str) -> None: + """ Sends a message to all registered rpc modules """ + def rpc_trade_status(self) -> Tuple[bool, Any]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 58e9bf2b9..ce01b78a3 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,12 +1,12 @@ """ This module contains class to manage RPC communications (Telegram, Slack, ...) """ -from typing import Any, List import logging +from typing import List +from freqtrade.rpc.rpc import RPC from freqtrade.rpc.telegram import Telegram - logger = logging.getLogger(__name__) @@ -15,36 +15,21 @@ class RPCManager(object): Class to manage RPC objects (Telegram, Slack, ...) """ def __init__(self, freqtrade) -> None: - """ - Initializes all enabled rpc modules - :param config: config to use - :return: None - """ - self.freqtrade = freqtrade + """ Initializes all enabled rpc modules """ + self.registered_modules: List[RPC] = [] - self.registered_modules: List[str] = [] - self.telegram: Any = None - self._init() - - def _init(self) -> None: - """ - Init RPC modules - :return: - """ - if self.freqtrade.config['telegram'].get('enabled', False): + # Enable telegram + if freqtrade.config['telegram'].get('enabled', False): logger.info('Enabling rpc.telegram ...') - self.registered_modules.append('telegram') - self.telegram = Telegram(self.freqtrade) + self.registered_modules.append(Telegram(freqtrade)) def cleanup(self) -> None: - """ - Stops all enabled rpc modules - :return: None - """ - if 'telegram' in self.registered_modules: - logger.info('Cleaning up rpc.telegram ...') - self.registered_modules.remove('telegram') - self.telegram.cleanup() + """ Stops all enabled rpc modules """ + for mod in self.registered_modules: + logger.info('Cleaning up rpc.%s ...', mod.name) + mod.cleanup() + + self.registered_modules = [] def send_msg(self, msg: str) -> None: """ @@ -52,6 +37,7 @@ class RPCManager(object): :param msg: message :return: None """ - logger.info(msg) - if 'telegram' in self.registered_modules: - self.telegram.send_msg(msg) + logger.info('Sending rpc message: %s', msg) + for mod in self.registered_modules: + logger.debug('Forwarding message to rpc.%s', mod.name) + mod.send_msg(msg) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 43383fe43..c00ba6a8e 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -14,7 +14,6 @@ from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ from freqtrade.rpc.rpc import RPC - logger = logging.getLogger(__name__) @@ -57,6 +56,11 @@ class Telegram(RPC): """ Telegram, this class send messages to Telegram """ + + @property + def name(self) -> str: + return "telegram" + def __init__(self, freqtrade) -> None: """ Init the Telegram call, and init the super class RPC @@ -120,6 +124,10 @@ class Telegram(RPC): self._updater.stop() + def send_msg(self, msg: str) -> None: + """ Send a message to telegram channel """ + self._send_msg(msg) + def is_enabled(self) -> bool: """ Returns True if the telegram module is activated, False otherwise @@ -146,10 +154,10 @@ class Telegram(RPC): # Fetch open trade (error, trades) = self.rpc_trade_status() if error: - self.send_msg(trades, bot=bot) + self._send_msg(trades, bot=bot) else: for trademsg in trades: - self.send_msg(trademsg, bot=bot) + self._send_msg(trademsg, bot=bot) @authorized_only def _status_table(self, bot: Bot, update: Update) -> None: @@ -163,12 +171,12 @@ class Telegram(RPC): # Fetch open trade (err, df_statuses) = self.rpc_status_table() if err: - self.send_msg(df_statuses, bot=bot) + self._send_msg(df_statuses, bot=bot) else: message = tabulate(df_statuses, headers='keys', tablefmt='simple') message = "
{}
".format(message) - self.send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) @authorized_only def _daily(self, bot: Bot, update: Update) -> None: @@ -189,7 +197,7 @@ class Telegram(RPC): self._config['fiat_display_currency'] ) if error: - self.send_msg(stats, bot=bot) + self._send_msg(stats, bot=bot) else: stats = tabulate(stats, headers=[ @@ -203,7 +211,7 @@ class Telegram(RPC): timescale, stats ) - self.send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) @authorized_only def _profit(self, bot: Bot, update: Update) -> None: @@ -219,7 +227,7 @@ class Telegram(RPC): self._config['fiat_display_currency'] ) if error: - self.send_msg(stats, bot=bot) + self._send_msg(stats, bot=bot) return # Message to display @@ -250,7 +258,7 @@ class Telegram(RPC): best_pair=stats['best_pair'], best_rate=stats['best_rate'] ) - self.send_msg(markdown_msg, bot=bot) + self._send_msg(markdown_msg, bot=bot) @authorized_only def _balance(self, bot: Bot, update: Update) -> None: @@ -259,7 +267,7 @@ class Telegram(RPC): """ (error, result) = self.rpc_balance(self._config['fiat_display_currency']) if error: - self.send_msg('`All balances are zero.`') + self._send_msg('`All balances are zero.`') return (currencys, total, symbol, value) = result @@ -274,7 +282,7 @@ class Telegram(RPC): output += "\n*Estimated Value*:\n" \ "\t`BTC: {0: .8f}`\n" \ "\t`{1}: {2: .2f}`\n".format(total, symbol, value) - self.send_msg(output) + self._send_msg(output) @authorized_only def _start(self, bot: Bot, update: Update) -> None: @@ -287,7 +295,7 @@ class Telegram(RPC): """ (error, msg) = self.rpc_start() if error: - self.send_msg(msg, bot=bot) + self._send_msg(msg, bot=bot) @authorized_only def _stop(self, bot: Bot, update: Update) -> None: @@ -299,7 +307,7 @@ class Telegram(RPC): :return: None """ (error, msg) = self.rpc_stop() - self.send_msg(msg, bot=bot) + self._send_msg(msg, bot=bot) @authorized_only def _reload_conf(self, bot: Bot, update: Update) -> None: @@ -326,7 +334,7 @@ class Telegram(RPC): trade_id = update.message.text.replace('/forcesell', '').strip() (error, message) = self.rpc_forcesell(trade_id) if error: - self.send_msg(message, bot=bot) + self._send_msg(message, bot=bot) return @authorized_only @@ -340,7 +348,7 @@ class Telegram(RPC): """ (error, trades) = self.rpc_performance() if error: - self.send_msg(trades, bot=bot) + self._send_msg(trades, bot=bot) return stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( @@ -350,7 +358,7 @@ class Telegram(RPC): count=trade['count'] ) for i, trade in enumerate(trades)) message = 'Performance:\n{}'.format(stats) - self.send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) @authorized_only def _count(self, bot: Bot, update: Update) -> None: @@ -363,7 +371,7 @@ class Telegram(RPC): """ (error, trades) = self.rpc_count() if error: - self.send_msg(trades, bot=bot) + self._send_msg(trades, bot=bot) return message = tabulate({ @@ -373,7 +381,7 @@ class Telegram(RPC): }, headers=['current', 'max', 'total stake'], tablefmt='simple') message = "
{}
".format(message) logger.debug(message) - self.send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) @authorized_only def _help(self, bot: Bot, update: Update) -> None: @@ -399,7 +407,7 @@ class Telegram(RPC): "*/help:* `This help message`\n" \ "*/version:* `Show version`" - self.send_msg(message, bot=bot) + self._send_msg(message, bot=bot) @authorized_only def _version(self, bot: Bot, update: Update) -> None: @@ -410,10 +418,10 @@ class Telegram(RPC): :param update: message update :return: None """ - self.send_msg('*Version:* `{}`'.format(__version__), bot=bot) + self._send_msg('*Version:* `{}`'.format(__version__), bot=bot) - def send_msg(self, msg: str, bot: Bot = None, - parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + def _send_msg(self, msg: str, bot: Bot = None, + parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: """ Send given markdown message :param msg: message diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 1d56dea3a..6c073a251 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -7,49 +7,35 @@ from copy import deepcopy from unittest.mock import MagicMock from freqtrade.rpc.rpc_manager import RPCManager -from freqtrade.rpc.telegram import Telegram from freqtrade.tests.conftest import log_has, get_patched_freqtradebot def test_rpc_manager_object() -> None: - """ - Test the Arguments object has the mandatory methods - :return: None - """ - assert hasattr(RPCManager, '_init') + """ Test the Arguments object has the mandatory methods """ assert hasattr(RPCManager, 'send_msg') assert hasattr(RPCManager, 'cleanup') def test__init__(mocker, default_conf) -> None: - """ - Test __init__() method - """ - init_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager._init', MagicMock()) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + """ Test __init__() method """ + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False - rpc_manager = RPCManager(freqtradebot) - assert rpc_manager.freqtrade == freqtradebot + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) assert rpc_manager.registered_modules == [] - assert rpc_manager.telegram is None - assert init_mock.call_count == 1 def test_init_telegram_disabled(mocker, default_conf, caplog) -> None: - """ - Test _init() method with Telegram disabled - """ + """ Test _init() method with Telegram disabled """ caplog.set_level(logging.DEBUG) conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - freqtradebot = get_patched_freqtradebot(mocker, conf) - rpc_manager = RPCManager(freqtradebot) + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) assert not log_has('Enabling rpc.telegram ...', caplog.record_tuples) assert rpc_manager.registered_modules == [] - assert rpc_manager.telegram is None def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: @@ -59,14 +45,12 @@ def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - rpc_manager = RPCManager(freqtradebot) + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert log_has('Enabling rpc.telegram ...', caplog.record_tuples) len_modules = len(rpc_manager.registered_modules) assert len_modules == 1 - assert 'telegram' in rpc_manager.registered_modules - assert isinstance(rpc_manager.telegram, Telegram) + assert 'telegram' in [mod.name for mod in rpc_manager.registered_modules] def test_cleanup_telegram_disabled(mocker, default_conf, caplog) -> None: @@ -99,11 +83,11 @@ def test_cleanup_telegram_enabled(mocker, default_conf, caplog) -> None: rpc_manager = RPCManager(freqtradebot) # Check we have Telegram as a registered modules - assert 'telegram' in rpc_manager.registered_modules + assert 'telegram' in [mod.name for mod in rpc_manager.registered_modules] rpc_manager.cleanup() assert log_has('Cleaning up rpc.telegram ...', caplog.record_tuples) - assert 'telegram' not in rpc_manager.registered_modules + assert 'telegram' not in [mod.name for mod in rpc_manager.registered_modules] assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 0919455ad..47ccf4243 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -258,7 +258,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: _init=MagicMock(), rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), _status_table=status_table, - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -296,7 +296,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _status_table=status_table, - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -341,7 +341,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -397,7 +397,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -465,7 +465,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -506,7 +506,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -604,7 +604,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) @@ -634,7 +634,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) @@ -656,7 +656,7 @@ def test_start_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -680,7 +680,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -705,7 +705,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -730,7 +730,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) @@ -898,7 +898,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) @@ -940,7 +940,7 @@ def test_performance_handle(default_conf, update, ticker, fee, mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -981,7 +981,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) @@ -1004,7 +1004,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', @@ -1047,7 +1047,7 @@ def test_help_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) @@ -1067,7 +1067,7 @@ def test_version_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) @@ -1090,12 +1090,12 @@ def test_send_msg(default_conf, mocker) -> None: telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = False - telegram.send_msg('test', bot) + telegram._send_msg('test', bot) assert not bot.method_calls bot.reset_mock() telegram._config['telegram']['enabled'] = True - telegram.send_msg('test', bot) + telegram._send_msg('test', bot) assert len(bot.method_calls) == 1 @@ -1113,7 +1113,7 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None: telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True - telegram.send_msg('test', bot) + telegram._send_msg('test', bot) # Bot should've tried to send it twice assert len(bot.method_calls) == 2 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5339ebc24..7a184eccc 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -57,7 +57,7 @@ def patch_RPCManager(mocker) -> MagicMock: :param mocker: mocker to patch RPCManager class :return: RPCManager.send_msg MagicMock to track if this method is called """ - mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) rpc_mock = mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) return rpc_mock From 4048859912c174822d1095ba94b3a2616bc11b73 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Jun 2018 04:52:50 +0200 Subject: [PATCH 33/59] rpc: remove tuple return madness --- freqtrade/rpc/rpc.py | 147 +++++++-------- freqtrade/rpc/telegram.py | 217 +++++++++++------------ freqtrade/tests/rpc/test_rpc.py | 139 ++++++--------- freqtrade/tests/rpc/test_rpc_manager.py | 4 +- freqtrade/tests/rpc/test_rpc_telegram.py | 8 +- 5 files changed, 223 insertions(+), 292 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 5ac78f2cb..2270214ab 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -5,7 +5,7 @@ import logging from abc import abstractmethod from datetime import datetime, timedelta, date from decimal import Decimal -from typing import Dict, Tuple, Any +from typing import Dict, Tuple, Any, List import arrow import sqlalchemy as sql @@ -20,6 +20,10 @@ from freqtrade.state import State logger = logging.getLogger(__name__) +class RPCException(Exception): + pass + + class RPC(object): """ RPC class can be used to have extra feature, like bot data, and access to DB data @@ -33,30 +37,29 @@ class RPC(object): self.freqtrade = freqtrade @abstractmethod - def cleanup(self) -> str: + def cleanup(self) -> None: """ Cleanup pending module resources """ @property @abstractmethod - def name(self) -> None: + def name(self) -> str: """ Returns the lowercase name of this module """ @abstractmethod def send_msg(self, msg: str) -> None: """ Sends a message to all registered rpc modules """ - def rpc_trade_status(self) -> Tuple[bool, Any]: + def _rpc_trade_status(self) -> List[str]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function - :return: """ # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self.freqtrade.state != State.RUNNING: - return True, '*Status:* `trader is not running`' + raise RPCException('*Status:* `trader is not running`') elif not trades: - return True, '*Status:* `no active trade`' + raise RPCException('*Status:* `no active trade`') else: result = [] for trade in trades: @@ -95,14 +98,14 @@ class RPC(object): ) if order else None, ) result.append(message) - return False, result + return result - def rpc_status_table(self) -> Tuple[bool, Any]: + def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self.freqtrade.state != State.RUNNING: - return True, '*Status:* `trader is not running`' + raise RPCException('*Status:* `trader is not running`') elif not trades: - return True, '*Status:* `no active order`' + raise RPCException('*Status:* `no active order`') else: trades_list = [] for trade in trades: @@ -118,20 +121,16 @@ class RPC(object): columns = ['ID', 'Pair', 'Since', 'Profit'] df_statuses = DataFrame.from_records(trades_list, columns=columns) df_statuses = df_statuses.set_index(columns[0]) - # The style used throughout is to return a tuple - # consisting of (error_occured?, result) - # Another approach would be to just return the - # result, or raise error - return False, df_statuses + return df_statuses - def rpc_daily_profit( + def _rpc_daily_profit( self, timescale: int, - stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: + stake_currency: str, fiat_display_currency: str) -> List[List[Any]]: today = datetime.utcnow().date() profit_days: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): - return True, '*Daily [n]:* `must be an integer greater than 0`' + raise RPCException('*Daily [n]:* `must be an integer greater than 0`') fiat = self.freqtrade.fiat_converter for day in range(0, timescale): @@ -148,7 +147,7 @@ class RPC(object): 'trades': len(trades) } - stats = [ + return [ [ key, '{value:.8f} {symbol}'.format( @@ -170,13 +169,10 @@ class RPC(object): ] for key, value in profit_days.items() ] - return False, stats - def rpc_trade_statistics( - self, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: - """ - :return: cumulative profit statistics. - """ + def _rpc_trade_statistics( + self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: + """ Returns cumulative profit statistics """ trades = Trade.query.order_by(Trade.id).all() profit_all_coin = [] @@ -214,7 +210,7 @@ class RPC(object): .order_by(sql.text('profit_sum DESC')).first() if not best_pair: - return True, '*Status:* `no closed trade`' + raise RPCException('*Status:* `no closed trade`') bp_pair, bp_rate = best_pair @@ -237,35 +233,29 @@ class RPC(object): fiat_display_currency ) num = float(len(durations) or 1) - return ( - False, - { - 'profit_closed_coin': profit_closed_coin, - 'profit_closed_percent': profit_closed_percent, - 'profit_closed_fiat': profit_closed_fiat, - 'profit_all_coin': profit_all_coin, - 'profit_all_percent': profit_all_percent, - 'profit_all_fiat': profit_all_fiat, - 'trade_count': len(trades), - 'first_trade_date': arrow.get(trades[0].open_date).humanize(), - 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), - 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], - 'best_pair': bp_pair, - 'best_rate': round(bp_rate * 100, 2) - } - ) + return { + 'profit_closed_coin': profit_closed_coin, + 'profit_closed_percent': profit_closed_percent, + 'profit_closed_fiat': profit_closed_fiat, + 'profit_all_coin': profit_all_coin, + 'profit_all_percent': profit_all_percent, + 'profit_all_fiat': profit_all_fiat, + 'trade_count': len(trades), + 'first_trade_date': arrow.get(trades[0].open_date).humanize(), + 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), + 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], + 'best_pair': bp_pair, + 'best_rate': round(bp_rate * 100, 2), + } - def rpc_balance(self, fiat_display_currency: str) -> Tuple[bool, Any]: - """ - :return: current account balance per crypto - """ + def _rpc_balance(self, fiat_display_currency: str) -> Tuple[List[Dict], float, str, float]: + """ Returns current account balance per crypto """ output = [] total = 0.0 for coin, balance in exchange.get_balances().items(): if not balance['total']: continue - rate = None if coin == 'BTC': rate = 1.0 else: @@ -285,32 +275,28 @@ class RPC(object): } ) if total == 0.0: - return True, '`All balances are zero.`' + raise RPCException('`All balances are zero.`') fiat = self.freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) - return False, (output, total, symbol, value) + return output, total, symbol, value - def rpc_start(self) -> Tuple[bool, str]: - """ - Handler for start. - """ + def _rpc_start(self) -> str: + """ Handler for start """ if self.freqtrade.state == State.RUNNING: - return True, '*Status:* `already running`' + return '*Status:* `already running`' self.freqtrade.state = State.RUNNING - return False, '`Starting trader ...`' + return '`Starting trader ...`' - def rpc_stop(self) -> Tuple[bool, str]: - """ - Handler for stop. - """ + def _rpc_stop(self) -> str: + """ Handler for stop """ if self.freqtrade.state == State.RUNNING: self.freqtrade.state = State.STOPPED - return False, '`Stopping trader ...`' + return '`Stopping trader ...`' - return True, '*Status:* `already stopped`' + return '*Status:* `already stopped`' def rpc_reload_conf(self) -> str: """ Handler for reload_conf. """ @@ -318,11 +304,10 @@ class RPC(object): return '*Status:* `Reloading config ...`' # FIX: no test for this!!!! - def rpc_forcesell(self, trade_id) -> Tuple[bool, Any]: + def _rpc_forcesell(self, trade_id) -> None: """ Handler for forcesell . Sells the given trade at current price - :return: error or None """ def _exec_forcesell(trade: Trade) -> None: # Check if there is there is an open order @@ -352,13 +337,13 @@ class RPC(object): # ---- EOF def _exec_forcesell ---- if self.freqtrade.state != State.RUNNING: - return True, '`trader is not running`' + raise RPCException('`trader is not running`') if trade_id == 'all': # Execute sell for all open orders for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): _exec_forcesell(trade) - return False, '' + return # Query for trade trade = Trade.query.filter( @@ -369,19 +354,18 @@ class RPC(object): ).first() if not trade: logger.warning('forcesell: Invalid argument received') - return True, 'Invalid argument.' + raise RPCException('Invalid argument.') _exec_forcesell(trade) Trade.session.flush() - return False, '' - def rpc_performance(self) -> Tuple[bool, Any]: + def _rpc_performance(self) -> List[Dict]: """ Handler for performance. Shows a performance statistic from finished trades """ if self.freqtrade.state != State.RUNNING: - return True, '`trader is not running`' + raise RPCException('`trader is not running`') pair_rates = Trade.session.query(Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum'), @@ -390,19 +374,14 @@ class RPC(object): .group_by(Trade.pair) \ .order_by(sql.text('profit_sum DESC')) \ .all() - trades = [] - for (pair, rate, count) in pair_rates: - trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) + return [ + {'pair': pair, 'profit': round(rate * 100, 2), 'count': count} + for pair, rate, count in pair_rates + ] - return False, trades - - def rpc_count(self) -> Tuple[bool, Any]: - """ - Returns the number of trades running - :return: None - """ + def _rpc_count(self) -> List[Trade]: + """ Returns the number of trades running """ if self.freqtrade.state != State.RUNNING: - return True, '`trader is not running`' + raise RPCException('`trader is not running`') - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - return False, trades + return Trade.query.filter(Trade.is_open.is_(True)).all() diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c00ba6a8e..f3cb65edc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -12,7 +12,7 @@ from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) @@ -151,13 +151,11 @@ class Telegram(RPC): self._status_table(bot, update) return - # Fetch open trade - (error, trades) = self.rpc_trade_status() - if error: - self._send_msg(trades, bot=bot) - else: - for trademsg in trades: - self._send_msg(trademsg, bot=bot) + try: + for trade_msg in self._rpc_trade_status(): + self._send_msg(trade_msg, bot=bot) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _status_table(self, bot: Bot, update: Update) -> None: @@ -168,15 +166,12 @@ class Telegram(RPC): :param update: message update :return: None """ - # Fetch open trade - (err, df_statuses) = self.rpc_status_table() - if err: - self._send_msg(df_statuses, bot=bot) - else: + try: + df_statuses = self._rpc_status_table() message = tabulate(df_statuses, headers='keys', tablefmt='simple') - message = "
{}
".format(message) - - self._send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg("
{}
".format(message), parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _daily(self, bot: Bot, update: Update) -> None: @@ -191,14 +186,12 @@ class Telegram(RPC): timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): timescale = 7 - (error, stats) = self.rpc_daily_profit( - timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] - ) - if error: - self._send_msg(stats, bot=bot) - else: + try: + stats = self._rpc_daily_profit( + timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) stats = tabulate(stats, headers=[ 'Day', @@ -207,11 +200,10 @@ class Telegram(RPC): ], tablefmt='simple') message = 'Daily Profit over the last {} days:\n
{}
'\ - .format( - timescale, - stats - ) + .format(timescale, stats) self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _profit(self, bot: Bot, update: Update) -> None: @@ -222,67 +214,65 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, stats) = self.rpc_trade_statistics( - self._config['stake_currency'], - self._config['fiat_display_currency'] - ) - if error: - self._send_msg(stats, bot=bot) - return + try: + stats = self._rpc_trade_statistics( + self._config['stake_currency'], + self._config['fiat_display_currency']) - # Message to display - markdown_msg = "*ROI:* Close trades\n" \ - "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ - "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ - "*ROI:* All trades\n" \ - "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ - "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ - "*Total Trade Count:* `{trade_count}`\n" \ - "*First Trade opened:* `{first_trade_date}`\n" \ - "*Latest Trade opened:* `{latest_trade_date}`\n" \ - "*Avg. Duration:* `{avg_duration}`\n" \ - "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ - .format( - coin=self._config['stake_currency'], - fiat=self._config['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) - self._send_msg(markdown_msg, bot=bot) + # Message to display + markdown_msg = "*ROI:* Close trades\n" \ + "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ + "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ + "*ROI:* All trades\n" \ + "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ + "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ + "*Total Trade Count:* `{trade_count}`\n" \ + "*First Trade opened:* `{first_trade_date}`\n" \ + "*Latest Trade opened:* `{latest_trade_date}`\n" \ + "*Avg. Duration:* `{avg_duration}`\n" \ + "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ + .format( + coin=self._config['stake_currency'], + fiat=self._config['fiat_display_currency'], + profit_closed_coin=stats['profit_closed_coin'], + profit_closed_percent=stats['profit_closed_percent'], + profit_closed_fiat=stats['profit_closed_fiat'], + profit_all_coin=stats['profit_all_coin'], + profit_all_percent=stats['profit_all_percent'], + profit_all_fiat=stats['profit_all_fiat'], + trade_count=stats['trade_count'], + first_trade_date=stats['first_trade_date'], + latest_trade_date=stats['latest_trade_date'], + avg_duration=stats['avg_duration'], + best_pair=stats['best_pair'], + best_rate=stats['best_rate'] + ) + self._send_msg(markdown_msg, bot=bot) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _balance(self, bot: Bot, update: Update) -> None: """ Handler for /balance """ - (error, result) = self.rpc_balance(self._config['fiat_display_currency']) - if error: - self._send_msg('`All balances are zero.`') - return + try: + currencys, total, symbol, value = \ + self._rpc_balance(self._config['fiat_display_currency']) + output = '' + for currency in currencys: + output += "*{currency}:*\n" \ + "\t`Available: {available: .8f}`\n" \ + "\t`Balance: {balance: .8f}`\n" \ + "\t`Pending: {pending: .8f}`\n" \ + "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) - (currencys, total, symbol, value) = result - output = '' - for currency in currencys: - output += "*{currency}:*\n" \ - "\t`Available: {available: .8f}`\n" \ - "\t`Balance: {balance: .8f}`\n" \ - "\t`Pending: {pending: .8f}`\n" \ - "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) - - output += "\n*Estimated Value*:\n" \ - "\t`BTC: {0: .8f}`\n" \ - "\t`{1}: {2: .2f}`\n".format(total, symbol, value) - self._send_msg(output) + output += "\n*Estimated Value*:\n" \ + "\t`BTC: {0: .8f}`\n" \ + "\t`{1}: {2: .2f}`\n".format(total, symbol, value) + self._send_msg(output, bot=bot) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _start(self, bot: Bot, update: Update) -> None: @@ -293,9 +283,8 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, msg) = self.rpc_start() - if error: - self._send_msg(msg, bot=bot) + msg = self._rpc_start() + self._send_msg(msg, bot=bot) @authorized_only def _stop(self, bot: Bot, update: Update) -> None: @@ -306,7 +295,7 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, msg) = self.rpc_stop() + msg = self._rpc_stop() self._send_msg(msg, bot=bot) @authorized_only @@ -332,10 +321,10 @@ class Telegram(RPC): """ trade_id = update.message.text.replace('/forcesell', '').strip() - (error, message) = self.rpc_forcesell(trade_id) - if error: - self._send_msg(message, bot=bot) - return + try: + self._rpc_forcesell(trade_id) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _performance(self, bot: Bot, update: Update) -> None: @@ -346,19 +335,18 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, trades) = self.rpc_performance() - if error: - self._send_msg(trades, bot=bot) - return - - stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( - index=i + 1, - pair=trade['pair'], - profit=trade['profit'], - count=trade['count'] - ) for i, trade in enumerate(trades)) - message = 'Performance:\n{}'.format(stats) - self._send_msg(message, parse_mode=ParseMode.HTML) + try: + trades = self._rpc_performance() + stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( + index=i + 1, + pair=trade['pair'], + profit=trade['profit'], + count=trade['count'] + ) for i, trade in enumerate(trades)) + message = 'Performance:\n{}'.format(stats) + self._send_msg(message, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _count(self, bot: Bot, update: Update) -> None: @@ -369,19 +357,18 @@ class Telegram(RPC): :param update: message update :return: None """ - (error, trades) = self.rpc_count() - if error: - self._send_msg(trades, bot=bot) - return - - message = tabulate({ - 'current': [len(trades)], - 'max': [self._config['max_open_trades']], - 'total stake': [sum((trade.open_rate * trade.amount) for trade in trades)] - }, headers=['current', 'max', 'total stake'], tablefmt='simple') - message = "
{}
".format(message) - logger.debug(message) - self._send_msg(message, parse_mode=ParseMode.HTML) + try: + trades = self._rpc_count() + message = tabulate({ + 'current': [len(trades)], + 'max': [self._config['max_open_trades']], + 'total stake': [sum((trade.open_rate * trade.amount) for trade in trades)] + }, headers=['current', 'max', 'total stake'], tablefmt='simple') + message = "
{}
".format(message) + logger.debug(message) + self._send_msg(message, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e), bot=bot) @authorized_only def _help(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 5cdd22c7a..5d7b56bc5 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -7,9 +7,11 @@ Unit test file for rpc/rpc.py from datetime import datetime from unittest.mock import MagicMock +import pytest + from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc.rpc import RPC, RPCException from freqtrade.state import State from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap @@ -41,19 +43,16 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, result) = rpc.rpc_trade_status() - assert error - assert 'trader is not running' in result + with pytest.raises(RPCException, match=r'.*trader is not running*'): + rpc._rpc_trade_status() freqtradebot.state = State.RUNNING - (error, result) = rpc.rpc_trade_status() - assert error - assert 'no active trade' in result + with pytest.raises(RPCException, match=r'.*no active trade*'): + rpc._rpc_trade_status() freqtradebot.create_trade() - (error, result) = rpc.rpc_trade_status() - assert not error - trade = result[0] + trades = rpc._rpc_trade_status() + trade = trades[0] result_message = [ '*Trade ID:* `1`\n' @@ -68,7 +67,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: '*Current Profit:* `-0.59%`\n' '*Open Order:* `(limit buy rem=0.00000000)`' ] - assert result == result_message + assert trades == result_message assert trade.find('[ETH/BTC]') >= 0 @@ -90,17 +89,15 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, result) = rpc.rpc_status_table() - assert error - assert '*Status:* `trader is not running`' in result + with pytest.raises(RPCException, match=r'.*\*Status:\* `trader is not running``*'): + rpc._rpc_status_table() freqtradebot.state = State.RUNNING - (error, result) = rpc.rpc_status_table() - assert error - assert '*Status:* `no active order`' in result + with pytest.raises(RPCException, match=r'.*\*Status:\* `no active order`*'): + rpc._rpc_status_table() freqtradebot.create_trade() - (error, result) = rpc.rpc_status_table() + result = rpc._rpc_status_table() assert 'just now' in result['Since'].all() assert 'ETH/BTC' in result['Pair'].all() assert '-0.59%' in result['Profit'].all() @@ -140,8 +137,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, # Try valid data update.message.text = '/daily 2' - (error, days) = rpc.rpc_daily_profit(7, stake_currency, fiat_display_currency) - assert not error + days = rpc._rpc_daily_profit(7, stake_currency, fiat_display_currency) assert len(days) == 7 for day in days: # [datetime.date(2018, 1, 11), '0.00000000 BTC', '0.000 USD'] @@ -154,9 +150,8 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, assert str(days[0][0]) == str(datetime.utcnow().date()) # Try invalid data - (error, days) = rpc.rpc_daily_profit(0, stake_currency, fiat_display_currency) - assert error - assert days.find('must be an integer greater than 0') >= 0 + with pytest.raises(RPCException, match=r'.*must be an integer greater than 0*'): + rpc._rpc_daily_profit(0, stake_currency, fiat_display_currency) def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, @@ -184,9 +179,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, rpc = RPC(freqtradebot) - (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) - assert error - assert stats.find('no closed trade') >= 0 + with pytest.raises(RPCException, match=r'.*no closed trade*'): + rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) # Create some test data freqtradebot.create_trade() @@ -219,8 +213,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, trade.close_date = datetime.utcnow() trade.is_open = False - (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) - assert not error + stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) @@ -281,8 +274,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, for trade in Trade.query.order_by(Trade.id).all(): trade.open_rate = None - (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) - assert not error + stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert prec_satoshi(stats['profit_closed_coin'], 0) assert prec_satoshi(stats['profit_closed_percent'], 0) assert prec_satoshi(stats['profit_closed_fiat'], 0) @@ -330,18 +322,16 @@ def test_rpc_balance_handle(default_conf, mocker): freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) - (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency']) - assert not error - (trade, x, y, z) = res - assert prec_satoshi(x, 12) - assert prec_satoshi(z, 180000) - assert 'USD' in y - assert len(trade) == 1 - assert 'BTC' in trade[0]['currency'] - assert prec_satoshi(trade[0]['available'], 10) - assert prec_satoshi(trade[0]['balance'], 12) - assert prec_satoshi(trade[0]['pending'], 2) - assert prec_satoshi(trade[0]['est_btc'], 12) + output, total, symbol, value = rpc._rpc_balance(default_conf['fiat_display_currency']) + assert prec_satoshi(total, 12) + assert prec_satoshi(value, 180000) + assert 'USD' in symbol + assert len(output) == 1 + assert 'BTC' in output[0]['currency'] + assert prec_satoshi(output[0]['available'], 10) + assert prec_satoshi(output[0]['balance'], 12) + assert prec_satoshi(output[0]['pending'], 2) + assert prec_satoshi(output[0]['est_btc'], 12) def test_rpc_start(mocker, default_conf) -> None: @@ -361,13 +351,11 @@ def test_rpc_start(mocker, default_conf) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, result) = rpc.rpc_start() - assert not error + result = rpc._rpc_start() assert '`Starting trader ...`' in result assert freqtradebot.state == State.RUNNING - (error, result) = rpc.rpc_start() - assert error + result = rpc._rpc_start() assert '*Status:* `already running`' in result assert freqtradebot.state == State.RUNNING @@ -389,13 +377,11 @@ def test_rpc_stop(mocker, default_conf) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING - (error, result) = rpc.rpc_stop() - assert not error + result = rpc._rpc_stop() assert '`Stopping trader ...`' in result assert freqtradebot.state == State.STOPPED - (error, result) = rpc.rpc_stop() - assert error + result = rpc._rpc_stop() assert '*Status:* `already stopped`' in result assert freqtradebot.state == State.STOPPED @@ -428,36 +414,26 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' + with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + rpc._rpc_forcesell(None) freqtradebot.state = State.RUNNING - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == 'Invalid argument.' + with pytest.raises(RPCException, match=r'.*Invalid argument.*'): + rpc._rpc_forcesell(None) - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' + rpc._rpc_forcesell('all') freqtradebot.create_trade() - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' + rpc._rpc_forcesell('all') - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' + rpc._rpc_forcesell('1') freqtradebot.state = State.STOPPED - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' + with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + rpc._rpc_forcesell(None) - (error, res) = rpc.rpc_forcesell('all') - assert error - assert res == '`trader is not running`' + with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + rpc._rpc_forcesell('all') freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 @@ -475,9 +451,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called # and trade amount is updated - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' + rpc._rpc_forcesell('1') assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount @@ -495,9 +469,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: } ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called - (error, res) = rpc.rpc_forcesell('2') - assert not error - assert res == '' + rpc._rpc_forcesell('2') assert cancel_order_mock.call_count == 2 assert trade.amount == amount @@ -511,9 +483,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'side': 'sell' } ) - (error, res) = rpc.rpc_forcesell('3') - assert not error - assert res == '' + rpc._rpc_forcesell('3') # status quo, no exchange calls assert cancel_order_mock.call_count == 2 @@ -550,8 +520,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, trade.close_date = datetime.utcnow() trade.is_open = False - (error, res) = rpc.rpc_performance() - assert not error + res = rpc._rpc_performance() assert len(res) == 1 assert res[0]['pair'] == 'ETH/BTC' assert res[0]['count'] == 1 @@ -576,14 +545,12 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) - (error, trades) = rpc.rpc_count() + trades = rpc._rpc_count() nb_trades = len(trades) - assert not error assert nb_trades == 0 # Create some test data freqtradebot.create_trade() - (error, trades) = rpc.rpc_count() + trades = rpc._rpc_count() nb_trades = len(trades) - assert not error assert nb_trades == 1 diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 6c073a251..805424d26 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -104,7 +104,7 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: rpc_manager = RPCManager(freqtradebot) rpc_manager.send_msg('test') - assert log_has('test', caplog.record_tuples) + assert log_has('Sending rpc message: test', caplog.record_tuples) assert telegram_mock.call_count == 0 @@ -119,5 +119,5 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: rpc_manager = RPCManager(freqtradebot) rpc_manager.send_msg('test') - assert log_has('test', caplog.record_tuples) + assert log_has('Sending rpc message: test', caplog.record_tuples) assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 47ccf4243..87e884110 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -60,9 +60,7 @@ def test__init__(default_conf, mocker) -> None: def test_init(default_conf, mocker, caplog) -> None: - """ - Test _init() method - """ + """ Test _init() method """ start_polling = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock(return_value=start_polling)) @@ -256,7 +254,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), + _rpc_trade_status=MagicMock(return_value=[1, 2, 3]), _status_table=status_table, _send_msg=msg_mock ) @@ -667,7 +665,7 @@ def test_start_handle(default_conf, update, mocker) -> None: assert freqtradebot.state == State.STOPPED telegram._start(bot=MagicMock(), update=update) assert freqtradebot.state == State.RUNNING - assert msg_mock.call_count == 0 + assert msg_mock.call_count == 1 def test_start_handle_already_running(default_conf, update, mocker) -> None: From 3787dad212f085deabe276aab0085e9e3e11988f Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Jun 2018 23:50:38 +0200 Subject: [PATCH 34/59] don't import python-telegram-bot at runtime if disabled in config --- freqtrade/rpc/rpc_manager.py | 2 +- freqtrade/rpc/telegram.py | 2 ++ freqtrade/tests/rpc/test_rpc.py | 22 +++++++++++----------- freqtrade/tests/test_freqtradebot.py | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index ce01b78a3..a2699c0e8 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -5,7 +5,6 @@ import logging from typing import List from freqtrade.rpc.rpc import RPC -from freqtrade.rpc.telegram import Telegram logger = logging.getLogger(__name__) @@ -21,6 +20,7 @@ class RPCManager(object): # Enable telegram if freqtrade.config['telegram'].get('enabled', False): logger.info('Enabling rpc.telegram ...') + from freqtrade.rpc.telegram import Telegram self.registered_modules.append(Telegram(freqtrade)) def cleanup(self) -> None: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f3cb65edc..08b45ffe4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -16,6 +16,8 @@ from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) +logger.debug('Included module rpc.telegram ...') + def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Callable[..., Any]: """ diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 5d7b56bc5..b49b7fdcb 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -31,7 +31,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -77,7 +77,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -110,7 +110,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -165,7 +165,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, ticker=MagicMock(return_value={'price_usd': 15000.0}), ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -241,7 +241,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, ticker=MagicMock(return_value={'price_usd': 15000.0}), ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -312,7 +312,7 @@ def test_rpc_balance_handle(default_conf, mocker): ticker=MagicMock(return_value={'price_usd': 15000.0}), ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -340,7 +340,7 @@ def test_rpc_start(mocker, default_conf) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -366,7 +366,7 @@ def test_rpc_stop(mocker, default_conf) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -392,7 +392,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) cancel_order_mock = MagicMock() mocker.patch.multiple( @@ -495,7 +495,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), @@ -533,7 +533,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: """ patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.freqtradebot.exchange', validate_pairs=MagicMock(), diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 7a184eccc..1d272428e 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -57,7 +57,7 @@ def patch_RPCManager(mocker) -> MagicMock: :param mocker: mocker to patch RPCManager class :return: RPCManager.send_msg MagicMock to track if this method is called """ - mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) rpc_mock = mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) return rpc_mock From 34e10a145c946eabc8be69050408f5c3e7440dbc Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 00:15:04 +0200 Subject: [PATCH 35/59] remove Telegram.is_enabled() because RPCManager manages lifecycles --- freqtrade/rpc/telegram.py | 17 ------- freqtrade/tests/rpc/test_rpc_telegram.py | 58 ++---------------------- 2 files changed, 4 insertions(+), 71 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 08b45ffe4..701c23a51 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -80,12 +80,7 @@ class Telegram(RPC): Initializes this module with the given config, registers all known command handlers and starts polling for message updates - :param config: config to use - :return: None """ - if not self.is_enabled(): - return - self._updater = Updater(token=self._config['telegram']['token'], workers=0) # Register command handler and start telegram message polling @@ -121,21 +116,12 @@ class Telegram(RPC): Stops all running telegram threads. :return: None """ - if not self.is_enabled(): - return - self._updater.stop() def send_msg(self, msg: str) -> None: """ Send a message to telegram channel """ self._send_msg(msg) - def is_enabled(self) -> bool: - """ - Returns True if the telegram module is activated, False otherwise - """ - return bool(self._config.get('telegram', {}).get('enabled', False)) - @authorized_only def _status(self, bot: Bot, update: Update) -> None: """ @@ -418,9 +404,6 @@ class Telegram(RPC): :param parse_mode: telegram parse mode :return: None """ - if not self.is_enabled(): - return - bot = bot or self._updater.bot keyboard = [['/daily', '/profit', '/balance'], diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 87e884110..68afa7587 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -32,6 +32,9 @@ class DummyCls(Telegram): super().__init__(freqtrade) self.state = {'called': False} + def _init(self): + pass + @authorized_only def dummy_handler(self, *args, **kwargs) -> None: """ @@ -78,21 +81,6 @@ def test_init(default_conf, mocker, caplog) -> None: assert log_has(message_str, caplog.record_tuples) -def test_init_disabled(default_conf, mocker, caplog) -> None: - """ - Test _init() method when Telegram is disabled - """ - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - Telegram(get_patched_freqtradebot(mocker, conf)) - - message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ - "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \ - "['count'], ['help'], ['version']]" - - assert not log_has(message_str, caplog.record_tuples) - - def test_cleanup(default_conf, mocker) -> None: """ Test cleanup() method @@ -101,44 +89,11 @@ def test_cleanup(default_conf, mocker) -> None: updater_mock.stop = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock) - # not enabled - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - telegram = Telegram(get_patched_freqtradebot(mocker, conf)) - telegram.cleanup() - assert telegram._updater is None - assert updater_mock.call_count == 0 - assert not hasattr(telegram._updater, 'stop') - assert updater_mock.stop.call_count == 0 - - # enabled - conf['telegram']['enabled'] = True - telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) telegram.cleanup() assert telegram._updater.stop.call_count == 1 -def test_is_enabled(default_conf, mocker) -> None: - """ - Test is_enabled() method - """ - mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) - - telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) - assert telegram.is_enabled() - - -def test_is_not_enabled(default_conf, mocker) -> None: - """ - Test is_enabled() method - """ - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - telegram = Telegram(get_patched_freqtradebot(mocker, conf)) - - assert not telegram.is_enabled() - - def test_authorized_only(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are authorized @@ -1087,11 +1042,6 @@ def test_send_msg(default_conf, mocker) -> None: freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) - telegram._config['telegram']['enabled'] = False - telegram._send_msg('test', bot) - assert not bot.method_calls - bot.reset_mock() - telegram._config['telegram']['enabled'] = True telegram._send_msg('test', bot) assert len(bot.method_calls) == 1 From 6c1bb7983bef16659d6c4a0ad5a58400909d6e3c Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 00:20:10 +0200 Subject: [PATCH 36/59] rpc: make freqtrade a private variable --- freqtrade/rpc/rpc.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2270214ab..0d81ae8ae 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -34,7 +34,7 @@ class RPC(object): :param freqtrade: Instance of a freqtrade bot :return: None """ - self.freqtrade = freqtrade + self._freqtrade = freqtrade @abstractmethod def cleanup(self) -> None: @@ -56,7 +56,7 @@ class RPC(object): """ # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('*Status:* `trader is not running`') elif not trades: raise RPCException('*Status:* `no active trade`') @@ -102,7 +102,7 @@ class RPC(object): def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('*Status:* `trader is not running`') elif not trades: raise RPCException('*Status:* `no active order`') @@ -132,7 +132,7 @@ class RPC(object): if not (isinstance(timescale, int) and timescale > 0): raise RPCException('*Daily [n]:* `must be an integer greater than 0`') - fiat = self.freqtrade.fiat_converter + fiat = self._freqtrade.fiat_converter for day in range(0, timescale): profitday = today - timedelta(days=day) trades = Trade.query \ @@ -216,7 +216,7 @@ class RPC(object): # FIX: we want to keep fiatconverter in a state/environment, # doing this will utilize its caching functionallity, instead we reinitialize it here - fiat = self.freqtrade.fiat_converter + fiat = self._freqtrade.fiat_converter # Prepare data to display profit_closed_coin = round(sum(profit_closed_coin), 8) profit_closed_percent = round(nan_to_num(mean(profit_closed_percent)) * 100, 2) @@ -277,23 +277,23 @@ class RPC(object): if total == 0.0: raise RPCException('`All balances are zero.`') - fiat = self.freqtrade.fiat_converter + fiat = self._freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) return output, total, symbol, value def _rpc_start(self) -> str: """ Handler for start """ - if self.freqtrade.state == State.RUNNING: + if self._freqtrade.state == State.RUNNING: return '*Status:* `already running`' - self.freqtrade.state = State.RUNNING + self._freqtrade.state = State.RUNNING return '`Starting trader ...`' def _rpc_stop(self) -> str: """ Handler for stop """ - if self.freqtrade.state == State.RUNNING: - self.freqtrade.state = State.STOPPED + if self._freqtrade.state == State.RUNNING: + self._freqtrade.state = State.STOPPED return '`Stopping trader ...`' return '*Status:* `already stopped`' @@ -333,10 +333,10 @@ class RPC(object): # Get current rate and execute sell current_rate = exchange.get_ticker(trade.pair, False)['bid'] - self.freqtrade.execute_sell(trade, current_rate) + self._freqtrade.execute_sell(trade, current_rate) # ---- EOF def _exec_forcesell ---- - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('`trader is not running`') if trade_id == 'all': @@ -364,7 +364,7 @@ class RPC(object): Handler for performance. Shows a performance statistic from finished trades """ - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('`trader is not running`') pair_rates = Trade.session.query(Trade.pair, @@ -381,7 +381,7 @@ class RPC(object): def _rpc_count(self) -> List[Trade]: """ Returns the number of trades running """ - if self.freqtrade.state != State.RUNNING: + if self._freqtrade.state != State.RUNNING: raise RPCException('`trader is not running`') return Trade.query.filter(Trade.is_open.is_(True)).all() From 83eb7a0a9d7b108f2f1626f759f1d9d3acc0b5cb Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 00:58:24 +0200 Subject: [PATCH 37/59] adjust logging a bit and add some comments --- freqtrade/rpc/rpc.py | 6 ++++++ freqtrade/rpc/rpc_manager.py | 3 ++- freqtrade/rpc/telegram.py | 12 +++--------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0d81ae8ae..10bbc854a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -21,6 +21,12 @@ logger = logging.getLogger(__name__) class RPCException(Exception): + """ + Should be raised with a rpc-formatted message in an _rpc_* method + if the required state is wrong, i.e.: + + raise RPCException('*Status:* `no active trade`') + """ pass diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index a2699c0e8..370d18176 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -25,8 +25,9 @@ class RPCManager(object): def cleanup(self) -> None: """ Stops all enabled rpc modules """ + logger.info('Cleaning up rpc modules ...') for mod in self.registered_modules: - logger.info('Cleaning up rpc.%s ...', mod.name) + logger.debug('Cleaning up rpc.%s ...', mod.name) mod.cleanup() self.registered_modules = [] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 701c23a51..e9091eb2b 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -26,9 +26,7 @@ def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Call :return: decorated function """ def wrapper(self, *args, **kwargs): - """ - Decorator logic - """ + """ Decorator logic """ update = kwargs.get('update') or args[1] # Reject unauthorized messages @@ -55,9 +53,7 @@ def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Call class Telegram(RPC): - """ - Telegram, this class send messages to Telegram - """ + """ This class handles all telegram communication """ @property def name(self) -> str: @@ -241,9 +237,7 @@ class Telegram(RPC): @authorized_only def _balance(self, bot: Bot, update: Update) -> None: - """ - Handler for /balance - """ + """ Handler for /balance """ try: currencys, total, symbol, value = \ self._rpc_balance(self._config['fiat_display_currency']) From e14c9e2090c9175c93ca8eecfa8c216b8bebcb7e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Jun 2018 13:30:48 +0200 Subject: [PATCH 38/59] fix potential cleanup issue --- freqtrade/rpc/rpc_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 370d18176..252bbcdd8 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -26,11 +26,11 @@ class RPCManager(object): def cleanup(self) -> None: """ Stops all enabled rpc modules """ logger.info('Cleaning up rpc modules ...') - for mod in self.registered_modules: + while self.registered_modules: + mod = self.registered_modules.pop() logger.debug('Cleaning up rpc.%s ...', mod.name) mod.cleanup() - - self.registered_modules = [] + del mod def send_msg(self, msg: str) -> None: """ From 92b0cbdc19ad3664cac637e9aa1c936760d48202 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:07 +0200 Subject: [PATCH 39/59] Update ccxt from 1.14.177 to 1.14.186 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2272d47b8..8414ba7d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.177 +ccxt==1.14.186 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From f404e0f5b39e14add3443e088956e5dc5b5da8ba Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:08 +0200 Subject: [PATCH 40/59] Update requests from 2.18.4 to 2.19.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8414ba7d9..d1dd93dc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 -requests==2.18.4 +requests==2.19.0 urllib3==1.22 wrapt==1.10.11 pandas==0.23.0 From 038acd3f5eca3de209547f8d0c035887c544e197 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:09 +0200 Subject: [PATCH 41/59] Update pandas from 0.23.0 to 0.23.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d1dd93dc9..e5ac314d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==2.1.0 requests==2.19.0 urllib3==1.22 wrapt==1.10.11 -pandas==0.23.0 +pandas==0.23.1 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 From 875408215bd87d364b0a09701590707fc4b18f57 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 13 Jun 2018 14:22:11 +0200 Subject: [PATCH 42/59] Update numpy from 1.14.4 to 1.14.5 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e5ac314d1..491297708 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.1 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.14.4 +numpy==1.14.5 TA-Lib==0.4.17 pytest==3.6.1 pytest-mock==1.10.0 From 46080f516897d9b005ffee8a540dccd431a3bc07 Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 13 Jun 2018 15:29:27 +0200 Subject: [PATCH 43/59] define _rpc_reload_conf as private method --- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/rpc/telegram.py | 4 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 10bbc854a..34802f920 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -304,9 +304,9 @@ class RPC(object): return '*Status:* `already stopped`' - def rpc_reload_conf(self) -> str: + def _rpc_reload_conf(self) -> str: """ Handler for reload_conf. """ - self.freqtrade.state = State.RELOAD_CONF + self._freqtrade.state = State.RELOAD_CONF return '*Status:* `Reloading config ...`' # FIX: no test for this!!!! diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e9091eb2b..4dd23971b 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -289,8 +289,8 @@ class Telegram(RPC): :param update: message update :return: None """ - msg = self.rpc_reload_conf() - self.send_msg(msg, bot=bot) + msg = self._rpc_reload_conf() + self._send_msg(msg, bot=bot) @authorized_only def _forcesell(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 68afa7587..f022c09e4 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -706,7 +706,7 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - send_msg=msg_mock + _send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) From 61f92b7460b1781c0a17a149b4fdbdf1f297f181 Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 31 Oct 2017 21:57:58 +0200 Subject: [PATCH 44/59] bugfix --- freqtrade/vendor/qtpylib/indicators.py | 50 ++++++++++---------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index ee1f14e1f..c4b955626 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -248,45 +248,34 @@ def crossed_below(series1, series2): def rolling_std(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods - try: - if min_periods == window: - return numpy_rolling_std(series, window, True) - else: - try: - return series.rolling(window=window, min_periods=min_periods).std() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).std() - except BaseException: - return pd.rolling_std(series, window=window, min_periods=min_periods) - + if min_periods == window and len(series) > window: + return numpy_rolling_std(series, window, True) + else: + try: + return series.rolling(window=window, min_periods=min_periods).std() + except BaseException: + return pd.Series(series).rolling(window=window, min_periods=min_periods).std() # --------------------------------------------- def rolling_mean(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods - try: - if min_periods == window: - return numpy_rolling_mean(series, window, True) - else: - try: - return series.rolling(window=window, min_periods=min_periods).mean() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).mean() - except BaseException: - return pd.rolling_mean(series, window=window, min_periods=min_periods) - + if min_periods == window and len(series) > window: + return numpy_rolling_mean(series, window, True) + else: + try: + return series.rolling(window=window, min_periods=min_periods).mean() + except BaseException: + return pd.Series(series).rolling(window=window, min_periods=min_periods).mean() # --------------------------------------------- def rolling_min(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: - try: - return series.rolling(window=window, min_periods=min_periods).min() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).min() + return series.rolling(window=window, min_periods=min_periods).min() except BaseException: - return pd.rolling_min(series, window=window, min_periods=min_periods) + return pd.Series(series).rolling(window=window, min_periods=min_periods).min() # --------------------------------------------- @@ -294,12 +283,9 @@ def rolling_min(series, window=14, min_periods=None): def rolling_max(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: - try: - return series.rolling(window=window, min_periods=min_periods).min() - except BaseException: - return pd.Series(series).rolling(window=window, min_periods=min_periods).min() + return series.rolling(window=window, min_periods=min_periods).min() except BaseException: - return pd.rolling_min(series, window=window, min_periods=min_periods) + return pd.Series(series).rolling(window=window, min_periods=min_periods).min() # --------------------------------------------- From e6e5c5daf02a88fa875f5445c5fece13aaf109ef Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 31 Oct 2017 21:58:03 +0200 Subject: [PATCH 45/59] added zlma --- freqtrade/vendor/qtpylib/indicators.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index c4b955626..22fac4e2a 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -552,6 +552,26 @@ def stoch(df, window=14, d=3, k=3, fast=False): return pd.DataFrame(index=df.index, data=data) +# --------------------------------------------- +def zlma(series, window=20, kind="ema"): + """ + John Ehlers' Zero lag (exponential) moving average + https://en.wikipedia.org/wiki/Zero_lag_exponential_moving_average + """ + lag = (window - 1) // 2 + series = 2 * series - series.shift(lag) + if kind in ['ewm', 'ema']: + return ema(series, lag) + elif kind == "hma": + return hma(series, lag) + return sma(series, lag) + +def zlema(series, window): + return zlma(series, window, kind="ema") +def zlsma(series, window): + return zlma(series, window, kind="sma") +def zlhma(series, window): + return zlma(series, window, kind="hma") # --------------------------------------------- From 6edb25f5c2f083060e68e274f23ddcdb25d23e03 Mon Sep 17 00:00:00 2001 From: ran Date: Fri, 12 Jan 2018 09:27:52 +0200 Subject: [PATCH 46/59] fixed heikenashi calculation --- freqtrade/vendor/qtpylib/indicators.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 22fac4e2a..9830e08fe 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -110,10 +110,13 @@ def heikinashi(bars): bars = bars.copy() bars['ha_close'] = (bars['open'] + bars['high'] + bars['low'] + bars['close']) / 4 + bars['ha_open'] = (bars['open'].shift(1) + bars['close'].shift(1)) / 2 bars.loc[:1, 'ha_open'] = bars['open'].values[0] - bars.loc[1:, 'ha_open'] = ( - (bars['ha_open'].shift(1) + bars['ha_close'].shift(1)) / 2)[1:] + for x in range(2): + bars.loc[1:, 'ha_open'] = ( + (bars['ha_open'].shift(1) + bars['ha_close'].shift(1)) / 2)[1:] + bars['ha_high'] = bars.loc[:, ['high', 'ha_open', 'ha_close']].max(axis=1) bars['ha_low'] = bars.loc[:, ['low', 'ha_open', 'ha_close']].min(axis=1) From d684ff5715271a2ef52bf2cd9b85783b5077d69f Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 13 Jun 2018 16:20:13 +0200 Subject: [PATCH 47/59] drop zlma implementation --- freqtrade/vendor/qtpylib/indicators.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 9830e08fe..e68932998 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -261,6 +261,7 @@ def rolling_std(series, window=200, min_periods=None): # --------------------------------------------- + def rolling_mean(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods if min_periods == window and len(series) > window: @@ -273,6 +274,7 @@ def rolling_mean(series, window=200, min_periods=None): # --------------------------------------------- + def rolling_min(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: @@ -556,27 +558,7 @@ def stoch(df, window=14, d=3, k=3, fast=False): return pd.DataFrame(index=df.index, data=data) # --------------------------------------------- -def zlma(series, window=20, kind="ema"): - """ - John Ehlers' Zero lag (exponential) moving average - https://en.wikipedia.org/wiki/Zero_lag_exponential_moving_average - """ - lag = (window - 1) // 2 - series = 2 * series - series.shift(lag) - if kind in ['ewm', 'ema']: - return ema(series, lag) - elif kind == "hma": - return hma(series, lag) - return sma(series, lag) -def zlema(series, window): - return zlma(series, window, kind="ema") -def zlsma(series, window): - return zlma(series, window, kind="sma") -def zlhma(series, window): - return zlma(series, window, kind="hma") - -# --------------------------------------------- def zscore(bars, window=20, stds=1, col='close'): """ get zscore of price """ From e600be4f568b8f3712e01a9381553ac618507611 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 19:43:33 +0200 Subject: [PATCH 48/59] Reduce force-sell verbosity --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5df4fb28a..dbbdd4b80 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -176,8 +176,8 @@ class Backtesting(object): close_index=sell_row.Index, open_at_end=True ) - logger.info('Force_selling still open trade %s with %s perc - %s', btr.pair, - btr.profit_percent, btr.profit_abs) + logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, + btr.profit_percent, btr.profit_abs) return btr return None From c0289ad8449b22800cc2272ac6180d486eca621f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 13 Jun 2018 19:53:12 +0200 Subject: [PATCH 49/59] use list comprehension to build list --- freqtrade/optimize/backtesting.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index dbbdd4b80..df68c1f31 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -114,14 +114,11 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - records = [] - print(results) - for index, trade_entry in results.iterrows(): - pass - records.append((trade_entry.pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - trade_entry.open_index - 1, trade_entry.trade_duration)) + records = [(trade_entry.pair, trade_entry.profit_percent, + trade_entry.open_time.timestamp(), + trade_entry.close_time.timestamp(), + trade_entry.open_index - 1, trade_entry.trade_duration) + for index, trade_entry in results.iterrows()] if records: logger.info('Dumping backtest results to %s', recordfilename) From ea805a8fb7e5f12e02d781955d86bd639f48424e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 14 Jun 2018 14:22:06 +0200 Subject: [PATCH 50/59] Update ccxt from 1.14.186 to 1.14.196 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 491297708..b1821f785 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.186 +ccxt==1.14.196 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 5c3e37412e2d9976252f41a9ebb84f65148ae42e Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 14 Jun 2018 21:20:16 +0200 Subject: [PATCH 51/59] update docs --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 127b4ee20..1efb46b43 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -21,7 +21,7 @@ If the 5 min and 1 min ticker for the crypto-currencies to test is not already in the `testdata` folder, backtesting will download them automatically. Testdata files will not be updated until you specify it. -The result of backtesting will confirm you if your bot as more chance to make a profit than a loss. +The result of backtesting will confirm you if your bot has better odds of making a profit than a loss. The backtesting is very easy with freqtrade. From 1e208e39b08f2dd1eb0016cda01305a4b42aaae5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:07 +0200 Subject: [PATCH 52/59] Update ccxt from 1.14.196 to 1.14.198 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b1821f785..bd5b3b10a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.196 +ccxt==1.14.198 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From e8fd11d6cecf825188a1bc2f2b6210dd8c1ffdba Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:08 +0200 Subject: [PATCH 53/59] Update requests from 2.19.0 to 2.19.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd5b3b10a..e8240fd97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 -requests==2.19.0 +requests==2.19.1 urllib3==1.22 wrapt==1.10.11 pandas==0.23.1 From a8d25266f962ac9989d570017b42c297b7325774 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:07 +0200 Subject: [PATCH 54/59] Update ccxt from 1.14.196 to 1.14.198 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b1821f785..bd5b3b10a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.196 +ccxt==1.14.198 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From af16830a38e10a9ec0395d2c1bb38239646ebf13 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 15 Jun 2018 14:23:08 +0200 Subject: [PATCH 55/59] Update requests from 2.19.0 to 2.19.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd5b3b10a..e8240fd97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 -requests==2.19.0 +requests==2.19.1 urllib3==1.22 wrapt==1.10.11 pandas==0.23.1 From c1f8f641e6a696b513a8eb81d9e246d86b861df0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 15 Jun 2018 10:45:19 +0300 Subject: [PATCH 56/59] remove use of hyperopt_conf.py --- docs/bot-usage.md | 5 ++- freqtrade/optimize/__init__.py | 10 ++---- freqtrade/optimize/hyperopt.py | 6 +--- freqtrade/tests/optimize/test_hyperopt.py | 7 ---- freqtrade/tests/optimize/test_optimize.py | 2 -- user_data/hyperopt_conf.py | 42 ----------------------- 6 files changed, 6 insertions(+), 66 deletions(-) delete mode 100644 user_data/hyperopt_conf.py diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 8079d9816..0ca99ec0a 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -160,9 +160,8 @@ the parameter `-l` or `--live`. ## Hyperopt commands -It is possible to use hyperopt for trading strategy optimization. -Hyperopt uses an internal json config return by `hyperopt_optimize_conf()` -located in `freqtrade/optimize/hyperopt_conf.py`. +To optimize your strategy, you can use hyperopt parameter hyperoptimization +to find optimal parameter values for your stategy. ``` usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index fc5d53114..867e8c7dc 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -11,8 +11,6 @@ from freqtrade import misc, constants from freqtrade.exchange import get_ticker_history from freqtrade.arguments import TimeRange -from user_data.hyperopt_conf import hyperopt_optimize_conf - logger = logging.getLogger(__name__) @@ -83,7 +81,7 @@ def load_tickerdata_file( def load_data(datadir: str, ticker_interval: str, - pairs: Optional[List[str]] = None, + pairs: List[str], refresh_pairs: Optional[bool] = False, timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]: """ @@ -92,14 +90,12 @@ def load_data(datadir: str, """ result = {} - _pairs = pairs or hyperopt_optimize_conf()['exchange']['pair_whitelist'] - # If the user force the refresh of pairs if refresh_pairs: logger.info('Download data for all pairs and store them in %s', datadir) - download_pairs(datadir, _pairs, ticker_interval, timerange=timerange) + download_pairs(datadir, pairs, ticker_interval, timerange=timerange) - for pair in _pairs: + for pair in pairs: pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) if pairdata: result[pair] = pairdata diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 878acc2dc..5acd05766 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -27,7 +27,6 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from user_data.hyperopt_conf import hyperopt_optimize_conf logger = logging.getLogger(__name__) @@ -596,11 +595,8 @@ def start(args: Namespace) -> None: # Monkey patch the configuration with hyperopt_conf.py configuration = Configuration(args) logger.info('Starting freqtrade in Hyperopt mode') + config = configuration.load_config() - optimize_config = hyperopt_optimize_conf() - config = configuration._load_common_config(optimize_config) - config = configuration._load_backtesting_config(config) - config = configuration._load_hyperopt_config(config) config['exchange']['key'] = '' config['exchange']['secret'] = '' diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 3edfe4393..4ef5762e1 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -23,8 +23,6 @@ def init_hyperopt(default_conf, mocker): global _HYPEROPT_INITIALIZED, _HYPEROPT if not _HYPEROPT_INITIALIZED: mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', - MagicMock(return_value=default_conf)) mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) _HYPEROPT = Hyperopt(default_conf) _HYPEROPT_INITIALIZED = True @@ -64,8 +62,6 @@ def test_start(mocker, default_conf, caplog) -> None: """ start_mock = MagicMock() mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', - MagicMock(return_value=default_conf)) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) args = [ @@ -182,7 +178,6 @@ def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) @@ -227,7 +222,6 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> conf.update({'epochs': 1}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) @@ -270,7 +264,6 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) StrategyResolver({'strategy': 'DefaultStrategy'}) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 3f358cfb8..bac8a6b36 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -326,8 +326,6 @@ def test_load_tickerdata_file() -> None: def test_init(default_conf, mocker) -> None: - conf = {'exchange': {'pair_whitelist': []}} - mocker.patch('freqtrade.optimize.hyperopt_optimize_conf', return_value=conf) assert {} == optimize.load_data( '', pairs=[], diff --git a/user_data/hyperopt_conf.py b/user_data/hyperopt_conf.py deleted file mode 100644 index c3a6e2a29..000000000 --- a/user_data/hyperopt_conf.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -File that contains the configuration for Hyperopt -""" - - -def hyperopt_optimize_conf() -> dict: - """ - This function is used to define which parameters Hyperopt must used. - The "pair_whitelist" is only used is your are using Hyperopt with MongoDB, - without MongoDB, Hyperopt will use the pair your have set in your config file. - :return: - """ - return { - 'max_open_trades': 3, - 'stake_currency': 'BTC', - 'stake_amount': 0.01, - "minimal_roi": { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, - }, - 'stoploss': -0.10, - "bid_strategy": { - "ask_last_balance": 0.0 - }, - "exchange": { - "name": "bittrex", - "pair_whitelist": [ - "ETH/BTC", - "LTC/BTC", - "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", - "XLM/BTC", - "NXT/BTC", - "POWR/BTC", - "ADA/BTC", - "XMR/BTC" - ] - } - } From 0c85febe76bb61cad1f7e376d7188fd634ac4f29 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 15 Jun 2018 10:59:09 +0300 Subject: [PATCH 57/59] remove all mongodb related code --- docs/bot-usage.md | 7 ++-- docs/hyperopt.md | 36 ------------------- docs/installation.md | 28 ++------------- freqtrade/arguments.py | 6 ---- freqtrade/configuration.py | 5 --- freqtrade/optimize/hyperopt.py | 36 ++++++------------- freqtrade/tests/optimize/test_hyperopt.py | 31 ---------------- .../tests/optimize/test_hyperopt_config.py | 16 --------- freqtrade/tests/test_configuration.py | 5 --- scripts/start-hyperopt-worker.py | 27 -------------- scripts/start-mongodb.py | 21 ----------- 11 files changed, 16 insertions(+), 202 deletions(-) delete mode 100644 freqtrade/tests/optimize/test_hyperopt_config.py delete mode 100755 scripts/start-hyperopt-worker.py delete mode 100755 scripts/start-mongodb.py diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0ca99ec0a..25fc78f0a 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -165,7 +165,7 @@ to find optimal parameter values for your stategy. ``` usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] - [--timerange TIMERANGE] [-e INT] [--use-mongodb] + [--timerange TIMERANGE] [-e INT] [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] optional arguments: @@ -175,11 +175,8 @@ optional arguments: --realistic-simulation uses max_open_trades from config to simulate real world limitations - --timerange TIMERANGE - specify what timerange of data to use. + --timerange TIMERANGE specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) - --use-mongodb parallelize evaluations with mongodb (requires mongod - in PATH) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate list. Default: all diff --git a/docs/hyperopt.md b/docs/hyperopt.md index a079e34df..2ad94896a 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -9,7 +9,6 @@ parameters with Hyperopt. - [Advanced Hyperopt notions](#advanced-notions) - [Understand the Guards and Triggers](#understand-the-guards-and-triggers) - [Execute Hyperopt](#execute-hyperopt) - - [Hyperopt with MongoDB](#hyperopt-with-mongoDB) - [Understand the hyperopts result](#understand-the-backtesting-result) ## Prepare Hyperopt @@ -194,41 +193,6 @@ Legal values are: - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` -### Hyperopt with MongoDB -Hyperopt with MongoDB, is like Hyperopt under steroids. As you saw by -executing the previous command is the execution takes a long time. -To accelerate it you can use hyperopt with MongoDB. - -To run hyperopt with MongoDb you will need 3 terminals. - -**Terminal 1: Start MongoDB** -```bash -cd -source .env/bin/activate -python3 scripts/start-mongodb.py -``` - -**Terminal 2: Start Hyperopt worker** -```bash -cd -source .env/bin/activate -python3 scripts/start-hyperopt-worker.py -``` - -**Terminal 3: Start Hyperopt with MongoDB** -```bash -cd -source .env/bin/activate -python3 ./freqtrade/main.py -c config.json hyperopt --use-mongodb -``` - -**Re-run an Hyperopt** -To re-run Hyperopt you have to delete the existing MongoDB table. -```bash -cd -rm -rf .hyperopt/mongodb/ -``` - ## Understand the hyperopts result Once Hyperopt is completed you can use the result to adding new buy signal. Given following result from hyperopt: diff --git a/docs/installation.md b/docs/installation.md index 9818529f6..b6688885b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -225,17 +225,7 @@ cd .. rm -rf ./ta-lib* ``` -#### 3. [Optional] Install MongoDB - -Install MongoDB if you plan to optimize your strategy with Hyperopt. - -```bash -sudo apt-get install mongodb-org -``` - -> Complete tutorial from Digital Ocean: [How to Install MongoDB on Ubuntu 16.04](https://www.digitalocean.com/community/tutorials/how-to-install-mongodb-on-ubuntu-16-04). - -#### 4. Install FreqTrade +#### 3. Install FreqTrade Clone the git repository: @@ -243,7 +233,7 @@ Clone the git repository: git clone https://github.com/freqtrade/freqtrade.git ``` -#### 5. Configure `freqtrade` as a `systemd` service +#### 4. Configure `freqtrade` as a `systemd` service From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. @@ -267,19 +257,7 @@ sudo loginctl enable-linger "$USER" brew install python3 git wget ta-lib ``` -#### 2. [Optional] Install MongoDB - -Install MongoDB if you plan to optimize your strategy with Hyperopt. - -```bash -curl -O https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.4.10.tgz -tar -zxvf mongodb-osx-ssl-x86_64-3.4.10.tgz -mkdir -p /env/mongodb -cp -R -n mongodb-osx-x86_64-3.4.10/ /env/mongodb -export PATH=/env/mongodb/bin:$PATH -``` - -#### 3. Install FreqTrade +#### 2. Install FreqTrade Clone the git repository: diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 331bb73a0..b392fb53e 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -203,12 +203,6 @@ class Arguments(object): type=int, metavar='INT', ) - parser.add_argument( - '--use-mongodb', - help='parallelize evaluations with mongodb (requires mongod in PATH)', - dest='mongodb', - action='store_true', - ) parser.add_argument( '-s', '--spaces', help='Specify which parameters to hyperopt. Space separate list. \ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 1f14df560..7c3a5eb4b 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -188,11 +188,6 @@ class Configuration(object): logger.info('Parameter --epochs detected ...') logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) - # If --mongodb is used we add it to the configuration - if 'mongodb' in self.args and self.args.mongodb: - config.update({'mongodb': self.args.mongodb}) - logger.info('Parameter --use-mongodb detected ...') - # If --spaces is used we add it to the configuration if 'spaces' in self.args and self.args.spaces: config.update({'spaces': self.args.spaces}) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5acd05766..978d1fe2c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -19,7 +19,6 @@ from typing import Dict, Any, Callable, Optional import numpy import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe -from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib @@ -507,32 +506,20 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - if self.config.get('mongodb'): - logger.info('Using mongodb ...') + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries logger.info( - 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' + 'Continuing with trials. Current: %d, Total: %d', + self.current_tries, + self.total_tries ) - db_name = 'freqtrade_hyperopt' - self.trials = MongoTrials( - arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), - exp_key='exp1' - ) - else: - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, self.signal_handler) - # read trials file if we have one - if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: - self.trials = self.read_trials() - - self.current_tries = len(self.trials.results) - self.total_tries += self.current_tries - logger.info( - 'Continuing with trials. Current: %d, Total: %d', - self.current_tries, - self.total_tries - ) - try: best_parameters = fmin( fn=self.generate_optimizer, @@ -588,7 +575,6 @@ def start(args: Namespace) -> None: """ # Remove noisy log messages - logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) # Initialize configuration diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 4ef5762e1..ce7cd77c8 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -247,7 +247,6 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'epochs': 1}) - conf.update({'mongodb': False}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) @@ -341,7 +340,6 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'epochs': 1}) - conf.update({'mongodb': False}) conf.update({'timerange': None}) conf.update({'spaces': 'all'}) @@ -353,35 +351,6 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: mock_fmin.assert_called_once() -def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None: - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - mock_mongotrials = mocker.patch( - 'freqtrade.optimize.hyperopt.MongoTrials', - return_value=create_trials(mocker) - ) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'mongodb': True}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) - mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - - hyperopt = Hyperopt(conf) - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - mock_mongotrials.assert_called_once() - mock_fmin.assert_called_once() - - -# test log_trials_result -# test buy_strategy_generator def populate_buy_trend -# test optimizer if 'ro_t1' in params - def test_format_results(init_hyperopt): """ Test Hyperopt.format_results() diff --git a/freqtrade/tests/optimize/test_hyperopt_config.py b/freqtrade/tests/optimize/test_hyperopt_config.py deleted file mode 100644 index aa9424826..000000000 --- a/freqtrade/tests/optimize/test_hyperopt_config.py +++ /dev/null @@ -1,16 +0,0 @@ -# pragma pylint: disable=missing-docstring,W0212 - -from user_data.hyperopt_conf import hyperopt_optimize_conf - - -def test_hyperopt_optimize_conf(): - hyperopt_conf = hyperopt_optimize_conf() - - assert "max_open_trades" in hyperopt_conf - assert "stake_currency" in hyperopt_conf - assert "stake_amount" in hyperopt_conf - assert "minimal_roi" in hyperopt_conf - assert "stoploss" in hyperopt_conf - assert "bid_strategy" in hyperopt_conf - assert "exchange" in hyperopt_conf - assert "pair_whitelist" in hyperopt_conf['exchange'] diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index caaddbf25..212df2e96 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -310,7 +310,6 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: arglist = [ 'hyperopt', '--epochs', '10', - '--use-mongodb', '--spaces', 'all', ] @@ -324,10 +323,6 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert log_has('Parameter --epochs detected ...', caplog.record_tuples) assert log_has('Will run Hyperopt with for 10 epochs ...', caplog.record_tuples) - assert 'mongodb' in config - assert config['mongodb'] is True - assert log_has('Parameter --use-mongodb detected ...', caplog.record_tuples) - assert 'spaces' in config assert config['spaces'] == ['all'] assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py deleted file mode 100755 index 8b0ae6326..000000000 --- a/scripts/start-hyperopt-worker.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -import multiprocessing -import os -import subprocess - -PROC_COUNT = multiprocessing.cpu_count() - 1 -DB_NAME = 'freqtrade_hyperopt' -WORK_DIR = os.path.join( - os.path.sep, - os.path.abspath(os.path.dirname(__file__)), - '..', '.hyperopt', 'worker' -) -if not os.path.exists(WORK_DIR): - os.makedirs(WORK_DIR) - -# Spawn workers -command = [ - 'hyperopt-mongo-worker', - '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), - '--poll-interval=0.1', - '--workdir={}'.format(WORK_DIR), -] -processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] - -# Join all workers -for proc in processes: - proc.wait() diff --git a/scripts/start-mongodb.py b/scripts/start-mongodb.py deleted file mode 100755 index 910ee9233..000000000 --- a/scripts/start-mongodb.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 - -import os -import subprocess - - -DB_PATH = os.path.join( - os.path.sep, - os.path.abspath(os.path.dirname(__file__)), - '..', '.hyperopt', 'mongodb' -) -if not os.path.exists(DB_PATH): - os.makedirs(DB_PATH) - -subprocess.Popen([ - 'mongod', - '--bind_ip=127.0.0.1', - '--port=1234', - '--nohttpinterface', - '--dbpath={}'.format(DB_PATH), -]).wait() From 7e2e7946c5b12abffd222420d74156208ef41e59 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 16 Jun 2018 09:06:14 +0300 Subject: [PATCH 58/59] also unit tests now need config.json --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 88121945f..3f041f5dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,16 +16,15 @@ install: - pip install --upgrade flake8 coveralls pytest-random-order mypy - pip install -r requirements.txt - pip install -e . +- cp config.json.example config.json jobs: include: - script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls - script: - - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting - script: - - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 - script: flake8 freqtrade - script: mypy freqtrade From 17801871b15ea76710528c8c30499bc9e8b8a314 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 16 Jun 2018 14:23:06 +0200 Subject: [PATCH 59/59] Update ccxt from 1.14.198 to 1.14.201 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e8240fd97..7e01b10aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.198 +ccxt==1.14.201 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1