From 067208bc9dab1f58c40c7abeab67ab1a11ac1ac6 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 24 Aug 2019 00:10:35 +0300 Subject: [PATCH 01/38] make backtesting an attribute of Hyperopt --- freqtrade/optimize/hyperopt.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cf4740f1c..62a6ab27b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -37,7 +37,7 @@ INITIAL_POINTS = 30 MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization -class Hyperopt(Backtesting): +class Hyperopt: """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -46,7 +46,9 @@ class Hyperopt(Backtesting): hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) + self.config = config + self.backtesting = Backtesting(self.config) + self.custom_hyperopt = HyperOptResolver(self.config).hyperopt self.custom_hyperoptloss = HyperOptLossResolver(self.config).hyperoptloss @@ -70,10 +72,10 @@ class Hyperopt(Backtesting): # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_buy_trend'): - self.advise_buy = self.custom_hyperopt.populate_buy_trend # type: ignore + self.backtesting.advise_buy = self.custom_hyperopt.populate_buy_trend # type: ignore if hasattr(self.custom_hyperopt, 'populate_sell_trend'): - self.advise_sell = self.custom_hyperopt.populate_sell_trend # type: ignore + self.backtesting.advise_sell = self.custom_hyperopt.populate_sell_trend # type: ignore # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): @@ -249,22 +251,22 @@ class Hyperopt(Backtesting): """ params = self.get_args(_params) if self.has_space('roi'): - self.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) + self.backtesting.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) + self.backtesting.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('sell'): - self.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) + self.backtesting.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) if self.has_space('stoploss'): - self.strategy.stoploss = params['stoploss'] + self.backtesting.strategy.stoploss = params['stoploss'] processed = load(self.tickerdata_pickle) min_date, max_date = get_timeframe(processed) - results = self.backtest( + results = self.backtesting.backtest( { 'stake_amount': self.config['stake_amount'], 'processed': processed, @@ -345,9 +347,9 @@ class Hyperopt(Backtesting): data = load_data( datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=self.config['exchange']['pair_whitelist'], - ticker_interval=self.ticker_interval, + ticker_interval=self.backtesting.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, + exchange=self.backtesting.exchange, timerange=timerange ) @@ -364,15 +366,15 @@ class Hyperopt(Backtesting): (max_date - min_date).days ) - self.strategy.advise_indicators = \ + self.backtesting.strategy.advise_indicators = \ self.custom_hyperopt.populate_indicators # type: ignore - preprocessed = self.strategy.tickerdata_to_dataframe(data) + preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) dump(preprocessed, self.tickerdata_pickle) # We don't need exchange instance anymore while running hyperopt - self.exchange = None # type: ignore + self.backtesting.exchange = None # type: ignore self.load_previous_results() From 667a6233108ab2a9ae53865493ee267d7b77921f Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 24 Aug 2019 00:10:55 +0300 Subject: [PATCH 02/38] adjust tests --- freqtrade/tests/optimize/test_hyperopt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index cfd75fd9d..7b525454c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -429,7 +429,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) - hyperopt.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -441,8 +441,8 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: assert dumper.called # Should be called twice, once for tickerdata, once to save evaluations assert dumper.call_count == 2 - assert hasattr(hyperopt, "advise_sell") - assert hasattr(hyperopt, "advise_buy") + assert hasattr(hyperopt.backtesting, "advise_sell") + assert hasattr(hyperopt.backtesting, "advise_buy") assert hasattr(hyperopt, "max_open_trades") assert hyperopt.max_open_trades == default_conf['max_open_trades'] assert hasattr(hyperopt, "position_stacking") @@ -488,7 +488,7 @@ def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", fill_missing=True)} - dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) + dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -502,7 +502,7 @@ def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", fill_missing=True)} - dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) + dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -538,7 +538,7 @@ def test_generate_optimizer(mocker, default_conf) -> None: backtest_result = pd.DataFrame.from_records(trades, columns=labels) mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.backtest', + 'freqtrade.optimize.hyperopt.Backtesting.backtest', MagicMock(return_value=backtest_result) ) mocker.patch( @@ -644,7 +644,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: }) hyperopt = Hyperopt(default_conf) - hyperopt.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -681,7 +681,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> }) hyperopt = Hyperopt(default_conf) - hyperopt.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() From 3f6eeda3f0e76627e9b62b672d59e7be46ef64f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Aug 2019 18:06:14 +0200 Subject: [PATCH 03/38] Reset stoploss_order_id when recreating fails --- freqtrade/freqtradebot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e5ecef8bf..e88b9db6a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -662,6 +662,7 @@ class FreqtradeBot(object): return False except DependencyException as exception: + trade.stoploss_order_id = None logger.warning('Unable to place a stoploss order on exchange: %s', exception) # If stoploss order is canceled for some reason we add it @@ -674,6 +675,7 @@ class FreqtradeBot(object): trade.stoploss_order_id = str(stoploss_order_id) return False except DependencyException as exception: + trade.stoploss_order_id = None logger.warning('Stoploss order was cancelled, ' 'but unable to recreate one: %s', exception) @@ -726,7 +728,8 @@ class FreqtradeBot(object): )['id'] trade.stoploss_order_id = str(stoploss_order_id) except DependencyException: - logger.exception(f"Could create trailing stoploss order " + trade.stoploss_order_id = None + logger.exception(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") def _check_and_execute_sell(self, trade: Trade, sell_rate: float, From 365b9c3e9c0f74de5a2ad7eacabd1c325396788a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Aug 2019 18:06:33 +0200 Subject: [PATCH 04/38] Add test to correctly handle unsuccessfull ordercreation --- freqtrade/tests/test_freqtradebot.py | 40 +++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 24d070d2d..dab7a9ff7 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1112,6 +1112,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, # Third case: when stoploss was set but it was canceled for some reason # should set a stoploss immediately and return False + caplog.clear() trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = 100 @@ -1127,6 +1128,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, # Fourth case: when stoploss is set and it is hit # should unset stoploss_order_id and return true # as a trade actually happened + caplog.clear() freqtrade.create_trades() trade = Trade.query.first() trade.is_open = True @@ -1152,6 +1154,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, ) freqtrade.handle_stoploss_on_exchange(trade) assert log_has('Unable to place a stoploss order on exchange: ', caplog) + assert trade.stoploss_order_id is None # Fifth case: get_order returns InvalidOrder # It should try to add stoploss order @@ -1163,6 +1166,41 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert stoploss_limit.call_count == 1 +def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, + markets, limit_buy_order, limit_sell_order) -> None: + # Sixth case: stoploss order was cancelled but couldn't create new one + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + sell=MagicMock(return_value={'id': limit_sell_order['id']}), + get_fee=fee, + markets=PropertyMock(return_value=markets), + get_order=MagicMock(return_value={'status': 'canceled'}), + stoploss_limit=MagicMock(side_effect=DependencyException()), + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + freqtrade.create_trades() + trade = Trade.query.first() + trade.is_open = True + trade.open_order_id = '12345' + trade.stoploss_order_id = 100 + assert trade + + assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert log_has_re(r'Stoploss order was cancelled, but unable to recreate one.*', caplog) + assert trade.stoploss_order_id is None + assert trade.is_open is True + + def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, markets, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set @@ -1324,7 +1362,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c mocker.patch("freqtrade.exchange.Exchange.stoploss_limit", side_effect=DependencyException()) freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 - assert log_has_re(r"Could create trailing stoploss order for pair ETH/BTC\..*", caplog) + assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, From 8a17615b5a96fd59ed39ba317e4acb741d0f881b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Aug 2019 19:41:11 +0200 Subject: [PATCH 05/38] move exceptionhandling from create_order() to calling functions --- freqtrade/exchange/exchange.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a8e974991..53723ad51 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -320,7 +320,7 @@ class Exchange(object): if (order_types.get("stoploss_on_exchange") and not self._ft_has.get("stoploss_on_exchange", False)): raise OperationalException( - 'On exchange stoploss is not supported for %s.' % self.name + f'On exchange stoploss is not supported for {self.name}.' ) def validate_order_time_in_force(self, order_time_in_force: Dict) -> None: @@ -469,11 +469,26 @@ class Exchange(object): params = self._params.copy() params.update({'stopPrice': stop_price}) - - order = self.create_order(pair, ordertype, 'sell', amount, rate, params) - logger.info('stoploss limit order added for %s. ' - 'stop price: %s. limit: %s', pair, stop_price, rate) - return order + try: + order = self.create_order(pair, ordertype, 'sell', amount, rate, params) + logger.info('stoploss limit order added for %s. ' + 'stop price: %s. limit: %s', pair, stop_price, rate) + return order + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create {ordertype} sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate}.' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to sell amount {amount} at rate {rate}.' + f'Message: {e}') from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e @retrier def get_balance(self, currency: str) -> float: From ea179a8e38a2dbaaf38e7b6a1904c174215aab83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 09:43:10 +0200 Subject: [PATCH 06/38] stoploss_limit shall not use `create_order()` It needs to handle exceptions differently --- freqtrade/exchange/exchange.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 53723ad51..7a384b7cc 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -470,7 +470,12 @@ class Exchange(object): params = self._params.copy() params.update({'stopPrice': stop_price}) try: - order = self.create_order(pair, ordertype, 'sell', amount, rate, params) + amount = self.symbol_amount_prec(pair, amount) + + rate = self.symbol_price_prec(pair, rate) + + order = self._api.create_order(pair, ordertype, 'sell', + amount, rate, params) logger.info('stoploss limit order added for %s. ' 'stop price: %s. limit: %s', pair, stop_price, rate) return order From defa1c027d911c348586fbf7c0419113a122d796 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 09:50:37 +0200 Subject: [PATCH 07/38] Move stoploss_limit to binance subclass --- freqtrade/exchange/binance.py | 52 ++++++++++++++++++++++++++++++++++ freqtrade/exchange/exchange.py | 48 ++++--------------------------- 2 files changed, 58 insertions(+), 42 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 18e754e3f..b63ef59c8 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -2,6 +2,9 @@ import logging from typing import Dict +import ccxt + +from freqtrade import DependencyException, OperationalException, TemporaryError from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) @@ -25,3 +28,52 @@ class Binance(Exchange): limit = min(list(filter(lambda x: limit <= x, limit_range))) return super().get_order_book(pair, limit) + + def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + """ + creates a stoploss limit order. + this stoploss-limit is binance-specific. + It may work with a limited number of other exchanges, but this has not been tested yet. + + """ + ordertype = "stop_loss_limit" + + stop_price = self.symbol_price_prec(pair, stop_price) + + # Ensure rate is less than stop price + if stop_price <= rate: + raise OperationalException( + 'In stoploss limit order, stop price should be more than limit price') + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + params = self._params.copy() + params.update({'stopPrice': stop_price}) + try: + amount = self.symbol_amount_prec(pair, amount) + + rate = self.symbol_price_prec(pair, rate) + + order = self._api.create_order(pair, ordertype, 'sell', + amount, rate, params) + logger.info('stoploss limit order added for %s. ' + 'stop price: %s. limit: %s', pair, stop_price, rate) + return order + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create {ordertype} sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate}.' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to sell amount {amount} at rate {rate}.' + f'Message: {e}') from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7a384b7cc..534e2ea65 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -450,50 +450,14 @@ class Exchange(object): def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: """ creates a stoploss limit order. - NOTICE: it is not supported by all exchanges. only binance is tested for now. - TODO: implementation maybe needs to be moved to the binance subclass + Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each + exchange's subclass. + The exception below should never raise, since we disallow + starting the bot in validate_ordertypes() + Note: Changes to this interface need to be applied to all sub-classes too. """ - ordertype = "stop_loss_limit" - stop_price = self.symbol_price_prec(pair, stop_price) - - # Ensure rate is less than stop price - if stop_price <= rate: - raise OperationalException( - 'In stoploss limit order, stop price should be more than limit price') - - if self._config['dry_run']: - dry_order = self.dry_run_order( - pair, ordertype, "sell", amount, stop_price) - return dry_order - - params = self._params.copy() - params.update({'stopPrice': stop_price}) - try: - amount = self.symbol_amount_prec(pair, amount) - - rate = self.symbol_price_prec(pair, rate) - - order = self._api.create_order(pair, ordertype, 'sell', - amount, rate, params) - logger.info('stoploss limit order added for %s. ' - 'stop price: %s. limit: %s', pair, stop_price, rate) - return order - except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create {ordertype} sell order on market {pair}.' - f'Tried to sell amount {amount} at rate {rate}.' - f'Message: {e}') from e - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not create {ordertype} sell order on market {pair}. ' - f'Tried to sell amount {amount} at rate {rate}.' - f'Message: {e}') from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e + raise OperationalException(f"stoploss_limit not implemented for {self.name}.") @retrier def get_balance(self, currency: str) -> float: From 067c122bf3e4e53d45eee4b48e6083a531ef5e66 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 09:52:14 +0200 Subject: [PATCH 08/38] Adapt test to use Binance class --- freqtrade/tests/test_freqtradebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index dab7a9ff7..1119157c4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2414,7 +2414,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -2454,7 +2454,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, freqtrade.process_maybe_execute_sell(trade) assert trade.stoploss_order_id is None assert trade.is_open is False - print(trade.sell_reason) assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value assert rpc_mock.call_count == 2 From 2c66b33fd107e0829cba668002d549a56bac4c56 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 09:57:21 +0200 Subject: [PATCH 09/38] Adapt some tests to use Binance subclass for stoplosslimit --- freqtrade/exchange/exchange.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 534e2ea65..6c93f9128 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -457,7 +457,7 @@ class Exchange(object): Note: Changes to this interface need to be applied to all sub-classes too. """ - raise OperationalException(f"stoploss_limit not implemented for {self.name}.") + raise OperationalException(f"stoploss_limit is not implemented for {self.name}.") @retrier def get_balance(self, currency: str) -> float: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index e453b5dca..0dc07e409 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1474,17 +1474,17 @@ def test_stoploss_limit_order(default_conf, mocker): # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) with pytest.raises(OperationalException): @@ -1493,6 +1493,12 @@ def test_stoploss_limit_order(default_conf, mocker): exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) +def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, 'bittrex') + with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + def test_stoploss_limit_order_dry_run(default_conf, mocker): api_mock = MagicMock() order_type = 'stop_loss_limit' From cbf09b5ad9781f42b0b836d45fbbd3936576c59f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 10:07:47 +0200 Subject: [PATCH 10/38] Improve docstring for Exception --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 14f0bb819..5ccc2ff3c 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -11,7 +11,7 @@ class DependencyException(Exception): class OperationalException(Exception): """ - Requires manual intervention. + Requires manual intervention and will usually stop the bot. This happens when an exchange returns an unexpected error during runtime or given configuration is invalid. """ From a4c8b5bf5da8228c3300e3d4c83c763e290b0a57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 10:08:06 +0200 Subject: [PATCH 11/38] Move binance-specific test to test_binance.py --- freqtrade/exchange/binance.py | 5 +- freqtrade/tests/exchange/test_binance.py | 90 +++++++++++++++++++++++ freqtrade/tests/exchange/test_exchange.py | 78 -------------------- 3 files changed, 93 insertions(+), 80 deletions(-) create mode 100644 freqtrade/tests/exchange/test_binance.py diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index b63ef59c8..5834f26cd 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -50,9 +50,10 @@ class Binance(Exchange): pair, ordertype, "sell", amount, stop_price) return dry_order - params = self._params.copy() - params.update({'stopPrice': stop_price}) try: + params = self._params.copy() + params.update({'stopPrice': stop_price}) + amount = self.symbol_amount_prec(pair, amount) rate = self.symbol_price_prec(pair, rate) diff --git a/freqtrade/tests/exchange/test_binance.py b/freqtrade/tests/exchange/test_binance.py new file mode 100644 index 000000000..4afb7fcc4 --- /dev/null +++ b/freqtrade/tests/exchange/test_binance.py @@ -0,0 +1,90 @@ +from random import randint +from unittest.mock import MagicMock + +import ccxt +import pytest + +from freqtrade import DependencyException, OperationalException, TemporaryError +from freqtrade.tests.conftest import get_patched_exchange + + +def test_stoploss_limit_order(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'stop_loss_limit' + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 + assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + + # test exception handling + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(TemporaryError): + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(OperationalException, match=r".*DeadBeef.*"): + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + +def test_stoploss_limit_order_dry_run(default_conf, mocker): + api_mock = MagicMock() + order_type = 'stop_loss_limit' + default_conf['dry_run'] = True + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert 'type' in order + + assert order['type'] == order_type + assert order['price'] == 220 + assert order['amount'] == 1 diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 0dc07e409..afd7b441a 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1436,62 +1436,6 @@ def test_get_fee(default_conf, mocker, exchange_name): 'get_fee', 'calculate_fee') -def test_stoploss_limit_order(default_conf, mocker): - api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop_loss_limit' - - api_mock.create_order = MagicMock(return_value={ - 'id': order_id, - 'info': { - 'foo': 'bar' - } - }) - - default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - - with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) - - api_mock.create_order.reset_mock() - - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - assert 'id' in order - assert 'info' in order - assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 - assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} - - # test exception handling - with pytest.raises(DependencyException): - api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - with pytest.raises(DependencyException): - api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - with pytest.raises(TemporaryError): - api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - with pytest.raises(OperationalException): - api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, 'bittrex') @@ -1499,29 +1443,7 @@ def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) -def test_stoploss_limit_order_dry_run(default_conf, mocker): - api_mock = MagicMock() - order_type = 'stop_loss_limit' - default_conf['dry_run'] = True - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - - with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) - - api_mock.create_order.reset_mock() - - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - assert 'id' in order - assert 'info' in order - assert 'type' in order - - assert order['type'] == order_type - assert order['price'] == 220 - assert order['amount'] == 1 def test_merge_ft_has_dict(default_conf, mocker): From 5e12b05424df197a5d724d9bb164b139f775076a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 10:13:35 +0200 Subject: [PATCH 12/38] Improve test coverage --- freqtrade/tests/exchange/test_exchange.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index afd7b441a..d9f350c7b 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -101,18 +101,21 @@ def test_destroy(default_conf, mocker, caplog): def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' - with pytest.raises( - OperationalException, - match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + with pytest.raises(OperationalException, + match=f"Exchange {default_conf['exchange']['name']} is not supported"): Exchange(default_conf) default_conf['exchange']['name'] = 'binance' - with pytest.raises( - OperationalException, - match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + with pytest.raises(OperationalException, + match=f"Exchange {default_conf['exchange']['name']} is not supported"): mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) Exchange(default_conf) + with pytest.raises(OperationalException, + match=r"Initialization of ccxt failed. Reason: DeadBeef"): + mocker.patch("ccxt.binance", MagicMock(side_effect=ccxt.BaseError("DeadBeef"))) + Exchange(default_conf) + def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) @@ -1436,16 +1439,12 @@ def test_get_fee(default_conf, mocker, exchange_name): 'get_fee', 'calculate_fee') - def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, 'bittrex') with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - - def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple('freqtrade.exchange.Exchange', _init_ccxt=MagicMock(return_value=MagicMock()), From 565a543b7b58437f5ecdca63bc96702b0f8c50f9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 10:34:56 +0200 Subject: [PATCH 13/38] Use ccxt base methods to round timeframe --- freqtrade/exchange/exchange.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a8e974991..6abf1f270 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -14,6 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple import arrow import ccxt import ccxt.async_support as ccxt_async +from ccxt.base.decimal_to_precision import ROUND_UP, ROUND_DOWN from pandas import DataFrame from freqtrade import (DependencyException, InvalidOrderException, @@ -824,11 +825,9 @@ def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime: """ if not date: date = datetime.now(timezone.utc) - timeframe_secs = timeframe_to_seconds(timeframe) - # Get offset based on timerame_secs - offset = date.timestamp() % timeframe_secs - # Subtract seconds passed since last offset - new_timestamp = date.timestamp() - offset + + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, + ROUND_DOWN) // 1000 return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) @@ -839,9 +838,8 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: :param date: date to use. Defaults to utcnow() :returns: date of next candle (with utc timezone) """ - prevdate = timeframe_to_prev_date(timeframe, date) - timeframe_secs = timeframe_to_seconds(timeframe) - - # Add one interval to previous candle - new_timestamp = prevdate.timestamp() + timeframe_secs + if not date: + date = datetime.now(timezone.utc) + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, + ROUND_UP) // 1000 return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) From 8f8acf5b06410b0c425aa6e66bb9f4de695ef320 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 10:43:37 +0200 Subject: [PATCH 14/38] Update ccxt to have this implemented --- requirements-common.txt | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-common.txt b/requirements-common.txt index 3d80c3ef5..bd3afd8de 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1068 +ccxt==1.18.1084 SQLAlchemy==1.3.7 python-telegram-bot==11.1.0 arrow==0.14.5 diff --git a/setup.py b/setup.py index 631c8b654..b48bddd56 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup(name='freqtrade', tests_require=['pytest', 'pytest-mock', 'pytest-cov'], install_requires=[ # from requirements-common.txt - 'ccxt>=1.18', + 'ccxt>=1.18.1080', 'SQLAlchemy', 'python-telegram-bot', 'arrow', @@ -76,7 +76,7 @@ setup(name='freqtrade', 'plot': plot, 'all': all_extra, 'jupyter': jupyter, - + }, include_package_data=True, zip_safe=False, From e603cca7a590b8cd5229a0c827bf5f812eab0fbc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 10:53:56 +0200 Subject: [PATCH 15/38] Testing with now() should not pass in date/time --- freqtrade/tests/exchange/test_exchange.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index e453b5dca..dad2d9c37 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1604,7 +1604,7 @@ def test_timeframe_to_prev_date(): assert timeframe_to_prev_date(interval, date) == result date = datetime.now(tz=timezone.utc) - assert timeframe_to_prev_date("5m", date) < date + assert timeframe_to_prev_date("5m") < date def test_timeframe_to_next_date(): @@ -1629,4 +1629,4 @@ def test_timeframe_to_next_date(): assert timeframe_to_next_date(interval, date) == result date = datetime.now(tz=timezone.utc) - assert timeframe_to_next_date("5m", date) > date + assert timeframe_to_next_date("5m") > date From 3232251fea8f05c309169c9449261dc720ea5366 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 15:01:27 +0200 Subject: [PATCH 16/38] Refactor downloading ohlcv from utils to history --- freqtrade/data/history.py | 29 +++++++++++++++++++++++++++++ freqtrade/utils.py | 32 ++++++++------------------------ 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 007357d9a..aff9f5c74 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -280,6 +280,35 @@ def download_pair_history(datadir: Optional[Path], return False +def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], + dl_path: Path, timerange: TimeRange, + erase=False) -> List[str]: + """ + Refresh stored ohlcv data for backtesting and hyperopt operations. + Used by freqtrade download-data + :return: Pairs not available + """ + pairs_not_available = [] + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in timeframes: + + dl_file = pair_data_filename(dl_path, pair, ticker_interval) + if erase and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + return pairs_not_available + + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ Get the maximum timeframe for the given backtest data diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 56e60ec82..162493a3f 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -2,13 +2,13 @@ import logging import sys from argparse import Namespace from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List import arrow from freqtrade.configuration import Configuration, TimeRange from freqtrade.configuration.directory_operations import create_userdata_dir -from freqtrade.data.history import download_pair_history +from freqtrade.data.history import refresh_backtest_ohlcv_data from freqtrade.exchange import available_exchanges from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -75,39 +75,23 @@ def start_download_data(args: Namespace) -> None: logger.info(f'About to download pairs: {config["pairs"]}, ' f'intervals: {config["timeframes"]} to {dl_path}') - pairs_not_available = [] + pairs_not_available: List[str] = [] try: # Init exchange exchange = ExchangeResolver(config['exchange']['name'], config).exchange - for pair in config["pairs"]: - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for ticker_interval in config["timeframes"]: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if config.get("erase") and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {ticker_interval}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=str(ticker_interval), - timerange=timerange) + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: - logger.info( - f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") # configuration.resolve_pairs_list() print(config) From da7da2ce524e0bd41576251af7d1a48ee71184ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 15:02:40 +0200 Subject: [PATCH 17/38] Change tests to split function --- freqtrade/tests/data/test_history.py | 43 +++++++++++++- freqtrade/tests/test_utils.py | 87 ++++++---------------------- 2 files changed, 61 insertions(+), 69 deletions(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 7360f3c1c..ec4a05e63 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -5,7 +5,7 @@ import os import uuid from pathlib import Path from shutil import copyfile -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import arrow import pytest @@ -17,6 +17,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, make_testdata_path, + refresh_backtest_ohlcv_data, trim_tickerlist) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json @@ -558,3 +559,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('5m')) assert len(caplog.record_tuples) == 0 + + +def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], dl_path=make_testdata_path(None), + timerange=timerange, erase=True + ) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype == 'date' + + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_no_markets(mocker, default_conf, caplog): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], + dl_path=make_testdata_path(None), + timerange=timerange, erase=False + ) + + assert dl_mock.call_count == 0 + assert "ETH/BTC" in unav_pairs + assert "XRP/BTC" in unav_pairs + assert log_has("Skipping pair ETH/BTC...", caplog) diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index d04e62b28..9e09fd298 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -1,5 +1,4 @@ import re -from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pytest @@ -70,74 +69,8 @@ def test_create_datadir(caplog, mocker): assert len(caplog.record_tuples) == 0 -def test_download_data(mocker, markets, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) - ) - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - mocker.patch.object(Path, "unlink", MagicMock()) - - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - "--erase", - ] - start_download_data(get_args(args)) - - assert dl_mock.call_count == 4 - assert dl_mock.call_args[1]['timerange'].starttype is None - assert dl_mock.call_args[1]['timerange'].stoptype is None - assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) - assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) - - -def test_download_data_days(mocker, markets, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) - ) - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - mocker.patch.object(Path, "unlink", MagicMock()) - - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - "--days", "20", - ] - - start_download_data(get_args(args)) - - assert dl_mock.call_count == 4 - assert dl_mock.call_args[1]['timerange'].starttype == 'date' - - assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) - - -def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) - ) - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - ] - start_download_data(get_args(args)) - - assert dl_mock.call_count == 0 - assert log_has("Skipping pair ETH/BTC...", caplog) - assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) - - def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', + dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -152,3 +85,21 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): start_download_data(get_args(args)) assert dl_mock.call_count == 1 + + +def test_download_data_no_markets(mocker, caplog): + dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20" + ] + start_download_data(get_args(args)) + assert dl_mock.call_args[1]['timerange'].starttype == "date" + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) From 513e84880e4a0e60b3f1e04edd8637908c874818 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 20:38:51 +0200 Subject: [PATCH 18/38] Don't escape ticks where it's not needed --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/optimize/hyperopt.py | 6 +++--- freqtrade/persistence.py | 4 ++-- freqtrade/tests/optimize/test_hyperopt.py | 4 ++-- freqtrade/tests/test_configuration.py | 10 +++++----- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e88b9db6a..8857b95da 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -216,7 +216,7 @@ class FreqtradeBot(object): if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.get_open_trades()) if open_trades >= self.config['max_open_trades']: - logger.warning('Can\'t open a new trade: max number of trades is reached') + logger.warning("Can't open a new trade: max number of trades is reached") return None return available_amount / (self.config['max_open_trades'] - open_trades) @@ -351,8 +351,8 @@ class FreqtradeBot(object): min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( - f'Can\'t open a new trade for {pair_s}: stake amount ' - f'is too small ({stake_amount} < {min_stake_amount})' + f"Can't open a new trade for {pair_s}: stake amount " + f"is too small ({stake_amount} < {min_stake_amount})" ) return False diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 62a6ab27b..9c3f085b6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -124,14 +124,14 @@ class Hyperopt: Save hyperopt trials to file """ if self.trials: - logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) + logger.info("Saving %d evaluations to '%s'", len(self.trials), self.trials_file) dump(self.trials, self.trials_file) def read_trials(self) -> List: """ Read hyperopt trials file """ - logger.info('Reading Trials from \'%s\'', self.trials_file) + logger.info("Reading Trials from '%s'", self.trials_file) trials = load(self.trials_file) self.trials_file.unlink() return trials @@ -379,7 +379,7 @@ class Hyperopt: self.load_previous_results() cpus = cpu_count() - logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') + logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) logger.info(f'Number of parallel jobs set as: {config_jobs}') diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c844bbc4c..dff7e4ff6 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -48,8 +48,8 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: try: engine = create_engine(db_url, **kwargs) except NoSuchModuleError: - raise OperationalException(f'Given value for db_url: \'{db_url}\' ' - f'is no valid database URL! (See {_SQL_DOCS_URL})') + raise OperationalException(f"Given value for db_url: '{db_url}' " + f"is no valid database URL! (See {_SQL_DOCS_URL})") session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 7b525454c..f9ebf552d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -381,7 +381,7 @@ def test_save_trials_saves_trials(mocker, hyperopt, caplog) -> None: hyperopt.save_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') - assert log_has('Saving 1 evaluations to \'{}\''.format(trials_file), caplog) + assert log_has("Saving 1 evaluations to '{}'".format(trials_file), caplog) mock_dump.assert_called_once() @@ -390,7 +390,7 @@ def test_read_trials_returns_trials_file(mocker, hyperopt, caplog) -> None: mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) hyperopt_trial = hyperopt.read_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') - assert log_has('Reading Trials from \'{}\''.format(trials_file), caplog) + assert log_has("Reading Trials from '{}'".format(trials_file), caplog) assert hyperopt_trial == trials mock_load.assert_called_once() diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 10ce7e8cf..d98ff9ad0 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -41,14 +41,14 @@ def test_load_config_invalid_pair(default_conf) -> None: def test_load_config_missing_attributes(default_conf) -> None: default_conf.pop('exchange') - with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): + with pytest.raises(ValidationError, match=r".*'exchange' is a required property.*"): validate_config_schema(default_conf) def test_load_config_incorrect_stake_amount(default_conf) -> None: default_conf['stake_amount'] = 'fake' - with pytest.raises(ValidationError, match=r'.*\'fake\' does not match \'unlimited\'.*'): + with pytest.raises(ValidationError, match=r".*'fake' does not match 'unlimited'.*"): validate_config_schema(default_conf) @@ -472,7 +472,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert 'spaces' in config assert config['spaces'] == ['all'] - assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog) + assert log_has("Parameter -s/--spaces detected: ['all']", caplog) assert "runmode" in config assert config['runmode'] == RunMode.HYPEROPT @@ -722,7 +722,7 @@ def test_load_config_default_exchange(all_conf) -> None: assert 'exchange' not in all_conf with pytest.raises(ValidationError, - match=r'\'exchange\' is a required property'): + match=r"'exchange' is a required property"): validate_config_schema(all_conf) @@ -736,7 +736,7 @@ def test_load_config_default_exchange_name(all_conf) -> None: assert 'name' not in all_conf['exchange'] with pytest.raises(ValidationError, - match=r'\'name\' is a required property'): + match=r"'name' is a required property"): validate_config_schema(all_conf) From bfc68ec792dd230eda42bd6b55c2014c6d9631e1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 25 Aug 2019 23:36:42 +0300 Subject: [PATCH 19/38] minor cleanup in Backtesting --- freqtrade/optimize/backtesting.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 568615b53..f558c82f7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -64,6 +64,12 @@ class Backtesting(object): self.config['dry_run'] = True self.strategylist: List[IStrategy] = [] + if "ticker_interval" not in self.config: + raise OperationalException("Ticker-interval needs to be set in either configuration " + "or as cli argument `--ticker-interval 5m`") + self.ticker_interval = str(self.config.get('ticker_interval')) + self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) + self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange self.fee = self.exchange.get_fee() @@ -89,12 +95,6 @@ class Backtesting(object): Load strategy into backtesting """ self.strategy = strategy - if "ticker_interval" not in self.config: - raise OperationalException("Ticker-interval needs to be set in either configuration " - "or as cli argument `--ticker-interval 5m`") - - self.ticker_interval = self.config.get('ticker_interval') - self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell # Set stoploss_on_exchange to false for backtesting, From e5da5f7fe79e7ae2a2957f22bce426b97445dd20 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2019 08:07:03 +0000 Subject: [PATCH 20/38] Bump mkdocs-material from 4.4.0 to 4.4.1 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 4.4.0 to 4.4.1. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/4.4.0...4.4.1) Signed-off-by: dependabot-preview[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index ce76d52e5..10ea068ad 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1 +1 @@ -mkdocs-material==4.4.0 \ No newline at end of file +mkdocs-material==4.4.1 \ No newline at end of file From 75e3d22043abb0e96a03e2f2201c39261b970de2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2019 08:07:07 +0000 Subject: [PATCH 21/38] Bump pytest from 5.1.0 to 5.1.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.1.0...5.1.1) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6436c60e4..a7b0d358d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ flake8==3.7.8 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 mypy==0.720 -pytest==5.1.0 +pytest==5.1.1 pytest-asyncio==0.10.0 pytest-cov==2.7.1 pytest-mock==1.10.4 From 6b233eb862d79eaca0dd9a6734d4e6e287bbb3da Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2019 08:07:24 +0000 Subject: [PATCH 22/38] Bump ccxt from 1.18.1068 to 1.18.1085 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.18.1068 to 1.18.1085. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.18.1068...1.18.1085) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 3d80c3ef5..c8a9c2f74 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1068 +ccxt==1.18.1085 SQLAlchemy==1.3.7 python-telegram-bot==11.1.0 arrow==0.14.5 From 6af51358025a9a318cc04c2122b856485ac9f3ce Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2019 08:07:30 +0000 Subject: [PATCH 23/38] Bump pandas from 0.25.0 to 0.25.1 Bumps [pandas](https://github.com/pandas-dev/pandas) from 0.25.0 to 0.25.1. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v0.25.0...v0.25.1) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d558b5b8..e5015d620 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ -r requirements-common.txt numpy==1.17.0 -pandas==0.25.0 +pandas==0.25.1 scipy==1.3.1 From 92011f82945f2e36fa73d0bcaaa1ea13fa20a301 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 26 Aug 2019 19:44:33 +0200 Subject: [PATCH 24/38] Introduce strategy_version variable --- freqtrade/resolvers/strategy_resolver.py | 4 ++++ freqtrade/strategy/default_strategy.py | 1 + freqtrade/strategy/interface.py | 5 +++++ freqtrade/tests/strategy/test_strategy.py | 25 +++++++++++++++++++++++ user_data/strategies/test_strategy.py | 7 +++++-- 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 37aa96b68..4e6ef154b 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -153,6 +153,10 @@ class StrategyResolver(IResolver): strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) + if any([x == 2 for x in [strategy._populate_fun_len, + strategy._buy_fun_len, + strategy._sell_fun_len]]): + strategy.strategy_version = 1 try: return import_strategy(strategy, config=config) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 5c7d50a65..51ef49193 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -13,6 +13,7 @@ class DefaultStrategy(IStrategy): Default Strategy provided by freqtrade bot. You can override it with your own strategy """ + strategy_version: int = 2 # Minimal ROI designed for the strategy minimal_roi = { diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 99f5f26de..4133d31a9 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -60,6 +60,11 @@ class IStrategy(ABC): stoploss -> float: optimal stoploss designed for the strategy ticker_interval -> str: value of the ticker interval to use for the strategy """ + # Strategy interface version + # Default to version 2 + # version 1 is the initial interface without metadata dict + # Version 2 populate_* include metadata dict + strategy_version: int = 2 _populate_fun_len: int = 0 _buy_fun_len: int = 0 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 240b83b8b..28acdea61 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -380,6 +380,31 @@ def test_call_deprecated_function(result, monkeypatch, default_conf): assert resolver.strategy._populate_fun_len == 2 assert resolver.strategy._buy_fun_len == 2 assert resolver.strategy._sell_fun_len == 2 + assert resolver.strategy.strategy_version == 1 + + indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) + assert isinstance(indicator_df, DataFrame) + assert 'adx' in indicator_df.columns + + buydf = resolver.strategy.advise_buy(result, metadata=metadata) + assert isinstance(buydf, DataFrame) + assert 'buy' in buydf.columns + + selldf = resolver.strategy.advise_sell(result, metadata=metadata) + assert isinstance(selldf, DataFrame) + assert 'sell' in selldf + + +def test_strategy_versioning(result, monkeypatch, default_conf): + default_conf.update({'strategy': 'DefaultStrategy'}) + resolver = StrategyResolver(default_conf) + metadata = {'pair': 'ETH/BTC'} + + # Make sure we are using a legacy function + assert resolver.strategy._populate_fun_len == 3 + assert resolver.strategy._buy_fun_len == 3 + assert resolver.strategy._sell_fun_len == 3 + assert resolver.strategy.strategy_version == 2 indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) assert isinstance(indicator_df, DataFrame) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index d8ff790b2..876a2845a 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -28,6 +28,9 @@ class TestStrategy(IStrategy): - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend, populate_sell_trend, hyperopt_space, buy_strategy_generator """ + # Strategy intervace version - allow new iterations of the strategy interface. + # Check the documentation or the Sample strategy to get the latest version. + strategy_version: int = 2 # Minimal ROI designed for the strategy. # This attribute will be overridden if the config file contains "minimal_roi" @@ -256,14 +259,14 @@ class TestStrategy(IStrategy): # Retrieve best bid and best ask # ------------------------------------ """ - # first check if dataprovider is available + # first check if dataprovider is available if self.dp: if self.dp.runmode in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] """ - + return dataframe def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From 0e62b8bd8502d70c9675bcbf0327fde076494461 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 26 Aug 2019 20:16:03 +0200 Subject: [PATCH 25/38] Update strategy_version to INTERFACE_VERSION --- freqtrade/resolvers/strategy_resolver.py | 2 +- freqtrade/strategy/default_strategy.py | 2 +- freqtrade/strategy/interface.py | 4 ++-- freqtrade/tests/strategy/test_strategy.py | 6 +++--- user_data/strategies/test_strategy.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 4e6ef154b..514e9f22b 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -156,7 +156,7 @@ class StrategyResolver(IResolver): if any([x == 2 for x in [strategy._populate_fun_len, strategy._buy_fun_len, strategy._sell_fun_len]]): - strategy.strategy_version = 1 + strategy.INTERFACE_VERSION = 1 try: return import_strategy(strategy, config=config) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 51ef49193..4907f20ed 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -13,7 +13,7 @@ class DefaultStrategy(IStrategy): Default Strategy provided by freqtrade bot. You can override it with your own strategy """ - strategy_version: int = 2 + INTERFACE_VERSION = 2 # Minimal ROI designed for the strategy minimal_roi = { diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4133d31a9..3f2478cc0 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -62,9 +62,9 @@ class IStrategy(ABC): """ # Strategy interface version # Default to version 2 - # version 1 is the initial interface without metadata dict + # Version 1 is the initial interface without metadata dict # Version 2 populate_* include metadata dict - strategy_version: int = 2 + INTERFACE_VERSION: int = 2 _populate_fun_len: int = 0 _buy_fun_len: int = 0 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 28acdea61..b1a6a6548 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -380,7 +380,7 @@ def test_call_deprecated_function(result, monkeypatch, default_conf): assert resolver.strategy._populate_fun_len == 2 assert resolver.strategy._buy_fun_len == 2 assert resolver.strategy._sell_fun_len == 2 - assert resolver.strategy.strategy_version == 1 + assert resolver.strategy.INTERFACE_VERSION == 1 indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) assert isinstance(indicator_df, DataFrame) @@ -395,7 +395,7 @@ def test_call_deprecated_function(result, monkeypatch, default_conf): assert 'sell' in selldf -def test_strategy_versioning(result, monkeypatch, default_conf): +def test_strategy_interface_versioning(result, monkeypatch, default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) resolver = StrategyResolver(default_conf) metadata = {'pair': 'ETH/BTC'} @@ -404,7 +404,7 @@ def test_strategy_versioning(result, monkeypatch, default_conf): assert resolver.strategy._populate_fun_len == 3 assert resolver.strategy._buy_fun_len == 3 assert resolver.strategy._sell_fun_len == 3 - assert resolver.strategy.strategy_version == 2 + assert resolver.strategy.INTERFACE_VERSION == 2 indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) assert isinstance(indicator_df, DataFrame) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 876a2845a..8e2bf8973 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -30,7 +30,7 @@ class TestStrategy(IStrategy): """ # Strategy intervace version - allow new iterations of the strategy interface. # Check the documentation or the Sample strategy to get the latest version. - strategy_version: int = 2 + INTERFACE_VERSION = 2 # Minimal ROI designed for the strategy. # This attribute will be overridden if the config file contains "minimal_roi" From d9c2b7d460dbb9a1eeee7fc53aa43822ecc420e3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 26 Aug 2019 22:31:24 +0300 Subject: [PATCH 26/38] fix fetching ticker_interval from strategy --- freqtrade/optimize/backtesting.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f558c82f7..4fba47243 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -64,12 +64,6 @@ class Backtesting(object): self.config['dry_run'] = True self.strategylist: List[IStrategy] = [] - if "ticker_interval" not in self.config: - raise OperationalException("Ticker-interval needs to be set in either configuration " - "or as cli argument `--ticker-interval 5m`") - self.ticker_interval = str(self.config.get('ticker_interval')) - self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange self.fee = self.exchange.get_fee() @@ -87,6 +81,12 @@ class Backtesting(object): # No strategy list specified, only one strategy self.strategylist.append(StrategyResolver(self.config).strategy) + if "ticker_interval" not in self.config: + raise OperationalException("Ticker-interval needs to be set in either configuration " + "or as cli argument `--ticker-interval 5m`") + self.ticker_interval = str(self.config.get('ticker_interval')) + self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) + # Load one (first) strategy self._set_strategy(self.strategylist[0]) From a504abf00c67d9f79904d265ddca3d5d72d9a305 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 27 Aug 2019 04:12:00 +0300 Subject: [PATCH 27/38] minor: improvements in confuguration.md --- docs/configuration.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2d21de942..22b112627 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -61,8 +61,9 @@ Mandatory parameters are marked as **Required**. | `order_time_in_force` | None | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). | `exchange.name` | | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.sandbox` | false | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details. -| `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. -| `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. +| `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. Keep it in secrete, do not disclose publicly. +| `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. Keep it in secrete, do not disclose publicly. +| `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. Keep it in secrete, do not disclose publicly. | `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). | `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). | `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) @@ -76,8 +77,8 @@ Mandatory parameters are marked as **Required**. | `pairlist.method` | StaticPairList | Use static or dynamic volume-based pairlist. [More information below](#dynamic-pairlists). | `pairlist.config` | None | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists). | `telegram.enabled` | true | **Required.** Enable or not the usage of Telegram. -| `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. -| `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. +| `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. Keep it in secrete, do not disclose publicly. +| `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. Keep it in secrete, do not disclose publicly. | `webhook.enabled` | false | Enable usage of Webhook notifications | `webhook.url` | false | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. | `webhook.webhookbuy` | false | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. From d66fb86449eccca42637177d0764f1a2eebe5ae2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Aug 2019 06:32:01 +0200 Subject: [PATCH 28/38] Add documentation for interface_version --- docs/strategy-customization.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 9e32ded18..5cf3784ef 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -36,9 +36,14 @@ A strategy file contains all the information needed to build a good strategy: - Minimal ROI recommended - Stoploss strongly recommended -The bot also include a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. +The bot includes a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. You can test it with the parameter: `--strategy TestStrategy` +Additionally, there is an attribute called `INTERFACE_VERSION`, which defines the version of the strategy interface the bot should use. +The current version is 2 - which is also the default when it's not set explicitly in the strategy. + +Future versions will require this to be set. + ```bash freqtrade --strategy AwesomeStrategy ``` From 51fbeed71f882902254b23d2459a92cffd35f1f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Aug 2019 06:41:07 +0200 Subject: [PATCH 29/38] Rename TestStrategy to SampleStrategy --- docs/backtesting.md | 10 +++++----- docs/data-analysis.md | 2 +- docs/strategy-customization.md | 15 +++++++-------- freqtrade/tests/optimize/test_backtesting.py | 6 +++--- freqtrade/tests/optimize/test_hyperopt.py | 2 +- freqtrade/tests/strategy/legacy_strategy.py | 2 +- freqtrade/tests/strategy/test_strategy.py | 12 ++++++------ freqtrade/tests/test_arguments.py | 2 +- .../notebooks/strategy_analysis_example.ipynb | 2 +- .../{test_strategy.py => sample_strategy.py} | 9 ++++----- 10 files changed, 30 insertions(+), 32 deletions(-) rename user_data/strategies/{test_strategy.py => sample_strategy.py} (98%) diff --git a/docs/backtesting.md b/docs/backtesting.md index 13d19f0ca..3da76c0ce 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -79,18 +79,18 @@ freqtrade backtesting --datadir user_data/data/bittrex-20180101 #### With a (custom) strategy file ```bash -freqtrade -s TestStrategy backtesting +freqtrade -s SampleStrategy backtesting ``` -Where `-s TestStrategy` refers to the class name within the strategy file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory. +Where `-s SampleStrategy` refers to the class name within the strategy file `sample_strategy.py` found in the `freqtrade/user_data/strategies` directory. #### Comparing multiple Strategies ```bash -freqtrade backtesting --strategy-list TestStrategy1 AwesomeStrategy --ticker-interval 5m +freqtrade backtesting --strategy-list SampleStrategy1 AwesomeStrategy --ticker-interval 5m ``` -Where `TestStrategy1` and `AwesomeStrategy` refer to class names of strategies. +Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies. #### Exporting trades to file @@ -103,7 +103,7 @@ The exported trades can be used for [further analysis](#further-backtest-result- #### Exporting trades to file specifying a custom filename ```bash -freqtrade backtesting --export trades --export-filename=backtest_teststrategy.json +freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json ``` #### Running backtest with smaller testset diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 2f077edb7..f6277cac2 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -139,7 +139,7 @@ You can override strategy settings as demonstrated below. # Define some constants ticker_interval = "5m" # Name of the strategy class -strategy_name = 'TestStrategy' +strategy_name = 'SampleStrategy' # Path to user data user_data_dir = 'user_data' # Location of the strategy diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 9e32ded18..a08304abb 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -24,7 +24,7 @@ strategy file will be updated on Github. Put your custom strategy file into the directory `user_data/strategies`. Best copy the test-strategy and modify this copy to avoid having bot-updates override your changes. -`cp user_data/strategies/test_strategy.py user_data/strategies/awesome-strategy.py` +`cp user_data/strategies/sample_strategy.py user_data/strategies/awesome-strategy.py` ### Anatomy of a strategy @@ -36,14 +36,14 @@ A strategy file contains all the information needed to build a good strategy: - Minimal ROI recommended - Stoploss strongly recommended -The bot also include a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. -You can test it with the parameter: `--strategy TestStrategy` +The bot also include a sample strategy called `SampleStrategy` you can update: `user_data/strategies/sample_strategy.py`. +You can test it with the parameter: `--strategy SampleStrategy` ```bash freqtrade --strategy AwesomeStrategy ``` -**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +**For the following section we will use the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py) file as reference.** !!! Note Strategies and Backtesting @@ -109,9 +109,8 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame return dataframe ``` - !!! Note "Want more indicator examples?" - Look into the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py).
+ Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py). Then uncomment indicators you need. ### Buy signal rules @@ -122,7 +121,7 @@ It's important to always return the dataframe without removing/modifying the col This will method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action". -Sample from `user_data/strategies/test_strategy.py`: +Sample from `user_data/strategies/sample_strategy.py`: ```python def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -152,7 +151,7 @@ It's important to always return the dataframe without removing/modifying the col This will method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action". -Sample from `user_data/strategies/test_strategy.py`: +Sample from `user_data/strategies/sample_strategy.py`: ```python def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5c942ab72..52eae9df0 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -330,7 +330,7 @@ def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> No patch_exchange(mocker) del default_conf['ticker_interval'] default_conf['strategy_list'] = ['DefaultStrategy', - 'TestStrategy'] + 'SampleStrategy'] mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) with pytest.raises(OperationalException): @@ -877,7 +877,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): '--disable-max-market-positions', '--strategy-list', 'DefaultStrategy', - 'TestStrategy', + 'SampleStrategy', ] args = get_args(args) start_backtesting(args) @@ -898,7 +898,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', - 'Running backtesting for Strategy TestStrategy', + 'Running backtesting for Strategy SampleStrategy', ] for line in exists: diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f9ebf552d..9583de510 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -254,7 +254,7 @@ def test_start_failure(mocker, default_conf, caplog) -> None: args = [ '--config', 'config.json', - '--strategy', 'TestStrategy', + '--strategy', 'SampleStrategy', 'hyperopt', '--epochs', '5' ] diff --git a/freqtrade/tests/strategy/legacy_strategy.py b/freqtrade/tests/strategy/legacy_strategy.py index 2cd13b791..af1b617a6 100644 --- a/freqtrade/tests/strategy/legacy_strategy.py +++ b/freqtrade/tests/strategy/legacy_strategy.py @@ -15,7 +15,7 @@ class TestStrategyLegacy(IStrategy): """ This is a test strategy using the legacy function headers, which will be removed in a future update. - Please do not use this as a template, but refer to user_data/strategy/TestStrategy.py + Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py for a uptodate version of this template. """ diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 240b83b8b..33c6e1128 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -61,27 +61,27 @@ def test_search_strategy(): def test_load_strategy(default_conf, result): - default_conf.update({'strategy': 'TestStrategy'}) + default_conf.update({'strategy': 'SampleStrategy'}) resolver = StrategyResolver(default_conf) assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) def test_load_strategy_base64(result, caplog, default_conf): - with open("user_data/strategies/test_strategy.py", "rb") as file: + with open("user_data/strategies/sample_strategy.py", "rb") as file: encoded_string = urlsafe_b64encode(file.read()).decode("utf-8") - default_conf.update({'strategy': 'TestStrategy:{}'.format(encoded_string)}) + default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)}) resolver = StrategyResolver(default_conf) assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) # Make sure strategy was loaded from base64 (using temp directory)!! - assert log_has_re(r"Using resolved strategy TestStrategy from '" - + tempfile.gettempdir() + r"/.*/TestStrategy\.py'\.\.\.", caplog) + assert log_has_re(r"Using resolved strategy SampleStrategy from '" + + tempfile.gettempdir() + r"/.*/SampleStrategy\.py'\.\.\.", caplog) def test_load_strategy_invalid_directory(result, caplog, default_conf): resolver = StrategyResolver(default_conf) extra_dir = Path.cwd() / 'some/path' - resolver._load_strategy('TestStrategy', config=default_conf, extra_dir=extra_dir) + resolver._load_strategy('SampleStrategy', config=default_conf, extra_dir=extra_dir) assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 24f11e32e..381457bbd 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -102,7 +102,7 @@ def test_parse_args_backtesting_custom() -> None: '--refresh-pairs-cached', '--strategy-list', 'DefaultStrategy', - 'TestStrategy' + 'SampleStrategy' ] call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == ['test_conf.json'] diff --git a/user_data/notebooks/strategy_analysis_example.ipynb b/user_data/notebooks/strategy_analysis_example.ipynb index 014f4ca90..89d71fe9d 100644 --- a/user_data/notebooks/strategy_analysis_example.ipynb +++ b/user_data/notebooks/strategy_analysis_example.ipynb @@ -52,7 +52,7 @@ "# Define some constants\n", "ticker_interval = \"5m\"\n", "# Name of the strategy class\n", - "strategy_name = 'TestStrategy'\n", + "strategy_name = 'SampleStrategy'\n", "# Path to user data\n", "user_data_dir = 'user_data'\n", "# Location of the strategy\n", diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/sample_strategy.py similarity index 98% rename from user_data/strategies/test_strategy.py rename to user_data/strategies/sample_strategy.py index d8ff790b2..db16dff79 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/sample_strategy.py @@ -11,10 +11,9 @@ import numpy # noqa # This class is a sample. Feel free to customize it. -class TestStrategy(IStrategy): - __test__ = False # pytest expects to find tests here because of the name +class SampleStrategy(IStrategy): """ - This is a test strategy to inspire you. + This is an example strategy to inspire you. More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md You can: @@ -256,14 +255,14 @@ class TestStrategy(IStrategy): # Retrieve best bid and best ask # ------------------------------------ """ - # first check if dataprovider is available + # first check if dataprovider is available if self.dp: if self.dp.runmode in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] """ - + return dataframe def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From 756f44fcbdb0395025468d4b3f4806fff99a5918 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 28 Aug 2019 00:20:32 +0300 Subject: [PATCH 30/38] highlight really important notifications --- docs/configuration.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 22b112627..fcd6c2bf6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -61,9 +61,9 @@ Mandatory parameters are marked as **Required**. | `order_time_in_force` | None | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). | `exchange.name` | | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.sandbox` | false | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details. -| `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. Keep it in secrete, do not disclose publicly. -| `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. Keep it in secrete, do not disclose publicly. -| `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. Keep it in secrete, do not disclose publicly. +| `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** +| `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** +| `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. ***Keep it in secrete, do not disclose publicly.*** | `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). | `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). | `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) @@ -77,8 +77,8 @@ Mandatory parameters are marked as **Required**. | `pairlist.method` | StaticPairList | Use static or dynamic volume-based pairlist. [More information below](#dynamic-pairlists). | `pairlist.config` | None | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists). | `telegram.enabled` | true | **Required.** Enable or not the usage of Telegram. -| `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. Keep it in secrete, do not disclose publicly. -| `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. Keep it in secrete, do not disclose publicly. +| `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** +| `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** | `webhook.enabled` | false | Enable usage of Webhook notifications | `webhook.url` | false | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. | `webhook.webhookbuy` | false | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. From 8923c02222d3887ac10b012a5825cedb19c13e5c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 28 Aug 2019 06:07:18 +0200 Subject: [PATCH 31/38] docstring wording --- user_data/strategies/sample_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_data/strategies/sample_strategy.py b/user_data/strategies/sample_strategy.py index db16dff79..4610a2e41 100644 --- a/user_data/strategies/sample_strategy.py +++ b/user_data/strategies/sample_strategy.py @@ -13,7 +13,7 @@ import numpy # noqa # This class is a sample. Feel free to customize it. class SampleStrategy(IStrategy): """ - This is an example strategy to inspire you. + This is a sample strategy to inspire you. More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md You can: From 68adfc6607ba790ad3d26d8b1ef332d3190f886c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Aug 2019 06:42:56 +0200 Subject: [PATCH 32/38] Init exchange before datadir ... --- freqtrade/configuration/configuration.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f5480c53a..e58ce0de1 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -160,6 +160,11 @@ class Configuration(object): Extract information for sys.argv and load directory configurations --user-data, --datadir """ + # Check exchange parameter here - otherwise `datadir` might be wrong. + if "exchange" in self.args and self.args.exchange: + config['exchange']['name'] = self.args.exchange + logger.info(f"Using exchange {config['exchange']['name']}") + if 'user_data_dir' in self.args and self.args.user_data_dir: config.update({'user_data_dir': self.args.user_data_dir}) elif 'user_data_dir' not in config: @@ -297,10 +302,6 @@ class Configuration(object): self._args_to_config(config, argname='days', logstring='Detected --days: {}') - if "exchange" in self.args and self.args.exchange: - config['exchange']['name'] = self.args.exchange - logger.info(f"Using exchange {config['exchange']['name']}") - def _process_runmode(self, config: Dict[str, Any]) -> None: if not self.runmode: @@ -361,7 +362,7 @@ class Configuration(object): config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: # Fall back to /dl_path/pairs.json - pairs_file = Path(config['datadir']) / config['exchange']['name'].lower() / "pairs.json" + pairs_file = Path(config['datadir']) / "pairs.json" if pairs_file.exists(): with pairs_file.open('r') as f: config['pairs'] = json_load(f) From 6b3d25b54b4f47ebd3b658aeaee3ff2501a23d14 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Aug 2019 06:45:20 +0200 Subject: [PATCH 33/38] Fix datadir init when used wiht --exchange --- freqtrade/tests/test_configuration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d98ff9ad0..b8bc62eb6 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -871,3 +871,4 @@ def test_pairlist_resolving_fallback(mocker): assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] assert config['exchange']['name'] == 'binance' + assert config['datadir'] == str(Path.cwd() / "user_data/data/binance") From cabe2910062610ee91768f7d111c9e8e1894705e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Aug 2019 06:54:28 +0200 Subject: [PATCH 34/38] Fix test-leakage by not copying config correctly --- freqtrade/configuration/configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e58ce0de1..b1bd3ef1c 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -4,6 +4,7 @@ This module contains the configuration class import logging import warnings from argparse import Namespace +from copy import deepcopy from pathlib import Path from typing import Any, Callable, Dict, List, Optional @@ -56,7 +57,7 @@ class Configuration(object): config: Dict[str, Any] = {} if not files: - return constants.MINIMAL_CONFIG.copy() + return deepcopy(constants.MINIMAL_CONFIG) # We expect here a list of config filenames for path in files: From b6b7dcd61c28722f87f34bffefd84c0982b995ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Aug 2019 06:59:19 +0200 Subject: [PATCH 35/38] Test NotImplemented is cought correctly --- freqtrade/tests/rpc/test_rpc_manager.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 468e3e8e5..d34d76524 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -115,6 +115,22 @@ def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] +def test_send_msg_webhook_CustomMessagetype(mocker, default_conf, caplog) -> None: + caplog.set_level(logging.DEBUG) + default_conf['telegram']['enabled'] = False + default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} + mocker.patch('freqtrade.rpc.webhook.Webhook.send_msg', + MagicMock(side_effect=NotImplementedError)) + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) + + assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] + rpc_manager.send_msg({'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'status': 'TestMessage'}) + assert log_has( + "Message type RPCMessageType.CUSTOM_NOTIFICATION not implemented by handler webhook.", + caplog) + + def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) From d977695d48594f7f7c10610ad9798398b2e9eae0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Aug 2019 07:02:26 +0200 Subject: [PATCH 36/38] Catch NotImplementedError when sending messages (RPC should not crash your bot!) --- freqtrade/rpc/rpc_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index fad532aa0..d6e7b174d 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -56,7 +56,10 @@ class RPCManager(object): logger.info('Sending rpc message: %s', msg) for mod in self.registered_modules: logger.debug('Forwarding message to rpc.%s', mod.name) - mod.send_msg(msg) + try: + mod.send_msg(msg) + except NotImplementedError: + logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.") def startup_messages(self, config, pairlist) -> None: if config.get('dry_run', False): From 75dc174c76992ae7d762e2183acd798f111435be Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Aug 2019 07:02:57 +0200 Subject: [PATCH 37/38] support all messagetypes in webhook --- freqtrade/rpc/webhook.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index bfc82b8d6..37ca466de 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -43,7 +43,9 @@ class Webhook(RPC): valuedict = self._config['webhook'].get('webhookbuy', None) elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: valuedict = self._config['webhook'].get('webhooksell', None) - elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: + elif msg['type'] in(RPCMessageType.STATUS_NOTIFICATION, + RPCMessageType.CUSTOM_NOTIFICATION, + RPCMessageType.WARNING_NOTIFICATION): valuedict = self._config['webhook'].get('webhookstatus', None) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) From d060d277457b26e1e7229e04ccb3d6a95ef6e90d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Aug 2019 07:05:22 +0200 Subject: [PATCH 38/38] Add test for all messagetypes --- freqtrade/tests/rpc/test_rpc_webhook.py | 33 ++++++++++++++----------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py index cc491d4dd..1c6c07e16 100644 --- a/freqtrade/tests/rpc/test_rpc_webhook.py +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -91,21 +91,24 @@ def test_send_msg(default_conf, mocker): assert (msg_mock.call_args[0][0]["value3"] == default_conf["webhook"]["webhooksell"]["value3"].format(**msg)) - # Test notification - msg = { - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': 'Unfilled sell order for BTC cancelled due to timeout' - } - msg_mock = MagicMock() - mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) - webhook.send_msg(msg) - assert msg_mock.call_count == 1 - assert (msg_mock.call_args[0][0]["value1"] == - default_conf["webhook"]["webhookstatus"]["value1"].format(**msg)) - assert (msg_mock.call_args[0][0]["value2"] == - default_conf["webhook"]["webhookstatus"]["value2"].format(**msg)) - assert (msg_mock.call_args[0][0]["value3"] == - default_conf["webhook"]["webhookstatus"]["value3"].format(**msg)) + for msgtype in [RPCMessageType.STATUS_NOTIFICATION, + RPCMessageType.WARNING_NOTIFICATION, + RPCMessageType.CUSTOM_NOTIFICATION]: + # Test notification + msg = { + 'type': msgtype, + 'status': 'Unfilled sell order for BTC cancelled due to timeout' + } + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook.send_msg(msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhookstatus"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhookstatus"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhookstatus"]["value3"].format(**msg)) def test_exception_send_msg(default_conf, mocker, caplog):