From d34da3f981496068928d85f6378be6b10b6678b5 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 2 May 2021 12:17:59 +0300 Subject: [PATCH 01/16] Revert "Add dataframe parameter to custom_stoploss() and custom_sell() methods." This reverts commit 595b8735f80df633834a4d8266694cdcb52287b8. # Conflicts: # freqtrade/optimize/backtesting.py # freqtrade/strategy/interface.py --- freqtrade/freqtradebot.py | 13 ++++++------- freqtrade/optimize/backtesting.py | 7 +++---- freqtrade/strategy/interface.py | 21 +++++++++------------ tests/strategy/test_default_strategy.py | 3 +-- tests/strategy/test_interface.py | 4 ++-- 5 files changed, 21 insertions(+), 27 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c2b15d23f..09a5ea746 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,7 +11,6 @@ from typing import Any, Dict, List, Optional import arrow from cachetools import TTLCache -from pandas import DataFrame from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency @@ -784,10 +783,10 @@ class FreqtradeBot(LoggingMixin): config_ask_strategy = self.config.get('ask_strategy', {}) - analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, - self.strategy.timeframe) if (config_ask_strategy.get('use_sell_signal', True) or config_ask_strategy.get('ignore_roi_if_buy_signal', False)): + analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, + self.strategy.timeframe) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df) @@ -814,13 +813,13 @@ class FreqtradeBot(LoggingMixin): # resulting in outdated RPC messages self._sell_rate_cache[trade.pair] = sell_rate - if self._check_and_execute_sell(analyzed_df, trade, sell_rate, buy, sell): + if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True else: logger.debug('checking sell') sell_rate = self.get_sell_rate(trade.pair, True) - if self._check_and_execute_sell(analyzed_df, trade, sell_rate, buy, sell): + if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True logger.debug('Found no sell signal for %s.', trade) @@ -951,13 +950,13 @@ class FreqtradeBot(LoggingMixin): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") - def _check_and_execute_sell(self, dataframe: DataFrame, trade: Trade, sell_rate: float, + def _check_and_execute_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: """ Check and execute sell """ should_sell = self.strategy.should_sell( - dataframe, trade, sell_rate, datetime.now(timezone.utc), buy, sell, + trade, sell_rate, datetime.now(timezone.utc), buy, sell, force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 899da03e4..80d816985 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -247,10 +247,9 @@ class Backtesting: else: return sell_row[OPEN_IDX] - def _get_sell_trade_entry(self, dataframe: DataFrame, trade: LocalTrade, - sell_row: Tuple) -> Optional[LocalTrade]: + def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: - sell = self.strategy.should_sell(dataframe, trade, sell_row[OPEN_IDX], # type: ignore + sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX], sell_row[SELL_IDX], low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX]) @@ -398,7 +397,7 @@ class Backtesting: for trade in open_trades[pair]: # also check the buying candle for sell conditions. - trade_entry = self._get_sell_trade_entry(processed[pair], trade, row) + trade_entry = self._get_sell_trade_entry(trade, row) # Sell occured if trade_entry: # logger.debug(f"{pair} - Backtesting sell {trade}") diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 63dcc75d9..c483e6afb 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -277,7 +277,7 @@ class IStrategy(ABC, HyperStrategyMixin): return True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, dataframe: DataFrame, **kwargs) -> float: + current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -300,8 +300,7 @@ class IStrategy(ABC, HyperStrategyMixin): return self.stoploss def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, dataframe: DataFrame, - **kwargs) -> Optional[Union[str, bool]]: + current_profit: float, **kwargs) -> Optional[Union[str, bool]]: """ Custom sell signal logic indicating that specified position should be sold. Returning a string or True from this method is equal to setting sell signal on a candle at specified @@ -539,8 +538,8 @@ class IStrategy(ABC, HyperStrategyMixin): else: return False - def should_sell(self, dataframe: DataFrame, trade: Trade, rate: float, date: datetime, - buy: bool, sell: bool, low: float = None, high: float = None, + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, + sell: bool, low: float = None, high: float = None, force_stoploss: float = 0) -> SellCheckTuple: """ This function evaluates if one of the conditions required to trigger a sell @@ -556,9 +555,8 @@ class IStrategy(ABC, HyperStrategyMixin): trade.adjust_min_max_rates(high or current_rate) - stoplossflag = self.stop_loss_reached(dataframe=dataframe, current_rate=current_rate, - trade=trade, current_time=date, - current_profit=current_profit, + stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, + current_time=date, current_profit=current_profit, force_stoploss=force_stoploss, high=high) # Set current rate to high for backtesting sell @@ -583,7 +581,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: custom_reason = strategy_safe_wrapper(self.custom_sell, default_retval=False)( pair=trade.pair, trade=trade, current_time=date, current_rate=current_rate, - current_profit=current_profit, dataframe=dataframe) + current_profit=current_profit) if custom_reason: sell_signal = SellType.CUSTOM_SELL if isinstance(custom_reason, str): @@ -620,7 +618,7 @@ class IStrategy(ABC, HyperStrategyMixin): # logger.debug(f"{trade.pair} - No sell signal.") return SellCheckTuple(sell_type=SellType.NONE) - def stop_loss_reached(self, dataframe: DataFrame, current_rate: float, trade: Trade, + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, force_stoploss: float, high: float = None) -> SellCheckTuple: """ @@ -638,8 +636,7 @@ class IStrategy(ABC, HyperStrategyMixin): )(pair=trade.pair, trade=trade, current_time=current_time, current_rate=current_rate, - current_profit=current_profit, - dataframe=dataframe) + current_profit=current_profit) # Sanity check - error cases will return None if stop_loss_value: # logger.info(f"{trade.pair} {stop_loss_value=} {current_profit=}") diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index a8862e9c9..ec7b3c33d 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -41,5 +41,4 @@ def test_default_strategy(result, fee): rate=20000, time_in_force='gtc', sell_reason='roi') is True assert strategy.custom_stoploss(pair='ETH/BTC', trade=trade, current_time=datetime.now(), - current_rate=20_000, current_profit=0.05, dataframe=None - ) == strategy.stoploss + current_rate=20_000, current_profit=0.05) == strategy.stoploss diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a241d7f43..182dde335 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -360,7 +360,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili now = arrow.utcnow().datetime sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit), trade=trade, current_time=now, current_profit=profit, - force_stoploss=0, high=None, dataframe=None) + force_stoploss=0, high=None) assert isinstance(sl_flag, SellCheckTuple) assert sl_flag.sell_type == expected if expected == SellType.NONE: @@ -371,7 +371,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit2), trade=trade, current_time=now, current_profit=profit2, - force_stoploss=0, high=None, dataframe=None) + force_stoploss=0, high=None) assert sl_flag.sell_type == expected2 if expected2 == SellType.NONE: assert sl_flag.sell_flag is False From dc6e702fecc0c7480b9c5291e290362d8527247b Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 2 May 2021 12:20:25 +0300 Subject: [PATCH 02/16] Pass current_time to confirm_trade_entry/confirm_trade_exit. --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/strategy/interface.py | 7 +++++-- tests/strategy/test_default_strategy.py | 6 ++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 09a5ea746..d2e6ed417 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -552,7 +552,7 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, - time_in_force=time_in_force): + time_in_force=time_in_force, current_time=datetime.utcnow()): logger.info(f"User requested abortion of buying {pair}") return False amount = self.exchange.amount_to_precision(pair, amount) @@ -1190,8 +1190,8 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, - time_in_force=time_in_force, - sell_reason=sell_reason.sell_reason): + time_in_force=time_in_force, sell_reason=sell_reason.sell_reason, + current_time=datetime.utcnow()): logger.info(f"User requested abortion of selling {trade.pair}") return False diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c483e6afb..66adc36ec 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -229,7 +229,7 @@ class IStrategy(ABC, HyperStrategyMixin): pass def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, **kwargs) -> bool: + time_in_force: str, current_time: datetime, **kwargs) -> bool: """ Called right before placing a buy order. Timing for this function is critical, so avoid doing heavy computations or @@ -244,6 +244,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param amount: Amount in target (quote) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the buy-order is placed on the exchange. False aborts the process @@ -251,7 +252,8 @@ class IStrategy(ABC, HyperStrategyMixin): return True def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float, - rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + rate: float, time_in_force: str, sell_reason: str, + current_time: datetime, **kwargs) -> bool: """ Called right before placing a regular sell order. Timing for this function is critical, so avoid doing heavy computations or @@ -270,6 +272,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param sell_reason: Sell reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', 'sell_signal', 'force_sell', 'emergency_sell'] + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the sell-order is placed on the exchange. False aborts the process diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index ec7b3c33d..92ac9f63a 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -36,9 +36,11 @@ def test_default_strategy(result, fee): ) assert strategy.confirm_trade_entry(pair='ETH/BTC', order_type='limit', amount=0.1, - rate=20000, time_in_force='gtc') is True + rate=20000, time_in_force='gtc', + current_time=datetime.utcnow()) is True assert strategy.confirm_trade_exit(pair='ETH/BTC', trade=trade, order_type='limit', amount=0.1, - rate=20000, time_in_force='gtc', sell_reason='roi') is True + rate=20000, time_in_force='gtc', sell_reason='roi', + current_time=datetime.utcnow()) is True assert strategy.custom_stoploss(pair='ETH/BTC', trade=trade, current_time=datetime.now(), current_rate=20_000, current_profit=0.05) == strategy.stoploss From cdfa6adbe55af690c7e7eb007a53f8a566ec0f84 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 2 May 2021 12:20:54 +0300 Subject: [PATCH 03/16] Store pair datafrmes in dataprovider for backtesting. --- freqtrade/data/dataprovider.py | 3 +++ freqtrade/optimize/backtesting.py | 16 +++++++++++----- tests/strategy/test_interface.py | 8 ++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b4dea0743..6cc32157e 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -173,3 +173,6 @@ class DataProvider: return self._pairlists.whitelist.copy() else: raise OperationalException("Dataprovider was not initialized with a pairlist provider.") + + def clear_cache(self): + self.__cached_pairs = {} diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 80d816985..73ad4e774 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -64,8 +64,8 @@ class Backtesting: self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - dataprovider = DataProvider(self.config, self.exchange) - IStrategy.dp = dataprovider + self.dataprovider = DataProvider(self.config, self.exchange) + IStrategy.dp = self.dataprovider if self.config.get('strategy_list', None): for strat in list(self.config['strategy_list']): @@ -96,7 +96,7 @@ class Backtesting: "PrecisionFilter not allowed for backtesting multiple strategies." ) - dataprovider.add_pairlisthandler(self.pairlists) + self.dataprovider.add_pairlisthandler(self.pairlists) self.pairlists.refresh_pairlist() if len(self.pairlists.whitelist) == 0: @@ -176,6 +176,7 @@ class Backtesting: Trade.use_db = False PairLocks.reset_locks() Trade.reset_trades() + self.dataprovider.clear_cache() def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]: """ @@ -266,7 +267,8 @@ class Backtesting: pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, rate=closerate, time_in_force=time_in_force, - sell_reason=sell.sell_reason): + sell_reason=sell.sell_reason, + current_time=sell_row[DATE_IDX].to_pydatetime()): return None trade.close(closerate, show_msg=False) @@ -286,7 +288,7 @@ class Backtesting: # Confirm trade entry: if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX], - time_in_force=time_in_force): + time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()): return None if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): @@ -348,6 +350,10 @@ class Backtesting: trades: List[LocalTrade] = [] self.prepare_backtest(enable_protections) + # Update dataprovider cache + for pair, dataframe in processed.items(): + self.dataprovider._set_cached_df(pair, self.timeframe, dataframe) + # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) data: Dict = self._get_ohlcv_as_lists(processed) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 182dde335..ded396779 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -399,27 +399,27 @@ def test_custom_sell(default_conf, fee, caplog) -> None: ) now = arrow.utcnow().datetime - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_flag is False assert res.sell_type == SellType.NONE strategy.custom_sell = MagicMock(return_value=True) - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_flag is True assert res.sell_type == SellType.CUSTOM_SELL assert res.sell_reason == 'custom_sell' strategy.custom_sell = MagicMock(return_value='hello world') - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_type == SellType.CUSTOM_SELL assert res.sell_flag is True assert res.sell_reason == 'hello world' caplog.clear() strategy.custom_sell = MagicMock(return_value='h' * 100) - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_type == SellType.CUSTOM_SELL assert res.sell_flag is True assert res.sell_reason == 'h' * 64 From 6af4de8fe86f66746a312144a9964cbbcc5ebb43 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 2 May 2021 12:25:43 +0300 Subject: [PATCH 04/16] Remove dataframe parameter from docs. --- docs/strategy-advanced.md | 22 +++++++++------------- docs/strategy-customization.md | 3 +-- freqtrade/strategy/interface.py | 1 - 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index eb322df9d..383b2a1a9 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -60,7 +60,8 @@ from freqtrade.strategy import IStrategy, timeframe_to_prev_date class AwesomeStrategy(IStrategy): def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, - current_profit: float, dataframe: DataFrame, **kwargs): + current_profit: float, **kwargs): + dataframe = self.dp.get_analyzed_dataframe(pair, self.timeframe) trade_open_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc) trade_row = dataframe.loc[dataframe['date'] == trade_open_date].squeeze() @@ -105,8 +106,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -156,8 +156,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. if current_time - timedelta(minutes=120) > trade.open_date_utc: @@ -183,8 +182,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: if pair in ('ETH/BTC', 'XRP/BTC'): return -0.10 @@ -210,8 +208,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: if current_profit < 0.04: return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss @@ -250,8 +247,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: # evaluate highest to lowest, so that highest possible stop is used if current_profit > 0.40: @@ -293,8 +289,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: # Default return value result = 1 @@ -302,6 +297,7 @@ class AwesomeStrategy(IStrategy): # Using current_time directly would only work in backtesting. Live/dry runs need time to # be rounded to previous candle to be used as dataframe index. Rounding must also be # applied to `trade.open_date(_utc)` if it is used for `dataframe` indexing. + dataframe = self.dp.get_analyzed_dataframe(pair, self.timeframe) current_time = timeframe_to_prev_date(self.timeframe, current_time) current_row = dataframe.loc[dataframe['date'] == current_time].squeeze() if 'atr' in current_row: diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 59bfbde48..6c62c1e86 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -631,8 +631,7 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: # once the profit has risen above 10%, keep the stoploss at 7% above the open price if current_profit > 0.10: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 66adc36ec..7483abf6d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -296,7 +296,6 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. - :param dataframe: Analyzed dataframe for this pair. Can contain future data in backtesting. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New stoploss value, relative to the currentrate """ From 6fb4d83ab3f2cb7b4402223b073ccb0a868fcb76 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 2 May 2021 16:35:06 +0300 Subject: [PATCH 05/16] Fix dataprovider in hyperopt. --- freqtrade/optimize/backtesting.py | 1 + freqtrade/optimize/hyperopt.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 73ad4e774..6d8b329bb 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -353,6 +353,7 @@ class Backtesting: # Update dataprovider cache for pair, dataframe in processed.items(): self.dataprovider._set_cached_df(pair, self.timeframe, dataframe) + self.strategy.dp = self.dataprovider # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e0a6d50a0..5e3d01047 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -372,8 +372,6 @@ class Hyperopt: self.backtesting.exchange._api_async = None # type: ignore # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore - self.backtesting.strategy.dp = None # type: ignore - IStrategy.dp = None # type: ignore cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") From 9b4f6b41a2ade4d1857fa0a6ed223967a965b26f Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Mon, 3 May 2021 09:18:38 +0300 Subject: [PATCH 06/16] Use correct datetime. --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d2e6ed417..b3379a462 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -552,7 +552,7 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, - time_in_force=time_in_force, current_time=datetime.utcnow()): + time_in_force=time_in_force, current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of buying {pair}") return False amount = self.exchange.amount_to_precision(pair, amount) @@ -1191,7 +1191,7 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, time_in_force=time_in_force, sell_reason=sell_reason.sell_reason, - current_time=datetime.utcnow()): + current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of selling {trade.pair}") return False From d344194b360bd453c4c6ea673a2724ef72b0e8ad Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Mon, 3 May 2021 09:47:58 +0300 Subject: [PATCH 07/16] Fix dataprovider in hyperopt. --- freqtrade/data/dataprovider.py | 141 +++++++++++++++++------------- freqtrade/optimize/backtesting.py | 4 +- 2 files changed, 79 insertions(+), 66 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 6cc32157e..731815572 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -45,40 +45,6 @@ class DataProvider: """ self._pairlists = pairlists - def refresh(self, - pairlist: ListPairsWithTimeframes, - helping_pairs: ListPairsWithTimeframes = None) -> None: - """ - Refresh data, called with each cycle - """ - if helping_pairs: - self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) - else: - self._exchange.refresh_latest_ohlcv(pairlist) - - @property - def available_pairs(self) -> ListPairsWithTimeframes: - """ - Return a list of tuples containing (pair, timeframe) for which data is currently cached. - Should be whitelist + open trades. - """ - return list(self._exchange._klines.keys()) - - def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: - """ - Get candle (OHLCV) data for the given pair as DataFrame - Please use the `available_pairs` method to verify which pairs are currently cached. - :param pair: pair to get the data for - :param timeframe: Timeframe to get data for - :param copy: copy dataframe before returning if True. - Use False only for read-only operations (where the dataframe is not modified) - """ - if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines((pair, timeframe or self._config['timeframe']), - copy=copy) - else: - return DataFrame() - def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame: """ Get stored historical candle (OHLCV) data @@ -123,35 +89,6 @@ class DataProvider: return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) - def market(self, pair: str) -> Optional[Dict[str, Any]]: - """ - Return market data for the pair - :param pair: Pair to get the data for - :return: Market data dict from ccxt or None if market info is not available for the pair - """ - return self._exchange.markets.get(pair) - - def ticker(self, pair: str): - """ - Return last ticker data from exchange - :param pair: Pair to get the data for - :return: Ticker dict from exchange or empty dict if ticker is not available for the pair - """ - try: - return self._exchange.fetch_ticker(pair) - except ExchangeError: - return {} - - def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: - """ - Fetch latest l2 orderbook data - Warning: Does a network request - so use with common sense. - :param pair: pair to get the data for - :param maximum: Maximum number of orderbook entries to query - :return: dict including bids/asks with a total of `maximum` entries. - """ - return self._exchange.fetch_l2_order_book(pair, maximum) - @property def runmode(self) -> RunMode: """ @@ -175,4 +112,82 @@ class DataProvider: raise OperationalException("Dataprovider was not initialized with a pairlist provider.") def clear_cache(self): + """ + Clear pair dataframe cache. + """ self.__cached_pairs = {} + + # Exchange functions + + def refresh(self, + pairlist: ListPairsWithTimeframes, + helping_pairs: ListPairsWithTimeframes = None) -> None: + """ + Refresh data, called with each cycle + """ + if self._exchange is None: + raise OperationalException('Exchange is not available to DataProvider.') + if helping_pairs: + self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) + else: + self._exchange.refresh_latest_ohlcv(pairlist) + + @property + def available_pairs(self) -> ListPairsWithTimeframes: + """ + Return a list of tuples containing (pair, timeframe) for which data is currently cached. + Should be whitelist + open trades. + """ + if self._exchange is None: + raise OperationalException('Exchange is not available to DataProvider.') + return list(self._exchange._klines.keys()) + + def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: + """ + Get candle (OHLCV) data for the given pair as DataFrame + Please use the `available_pairs` method to verify which pairs are currently cached. + :param pair: pair to get the data for + :param timeframe: Timeframe to get data for + :param copy: copy dataframe before returning if True. + Use False only for read-only operations (where the dataframe is not modified) + """ + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + return self._exchange.klines((pair, timeframe or self._config['timeframe']), + copy=copy) + else: + return DataFrame() + + def market(self, pair: str) -> Optional[Dict[str, Any]]: + """ + Return market data for the pair + :param pair: Pair to get the data for + :return: Market data dict from ccxt or None if market info is not available for the pair + """ + if self._exchange is None: + raise OperationalException('Exchange is not available to DataProvider.') + return self._exchange.markets.get(pair) + + def ticker(self, pair: str): + """ + Return last ticker data from exchange + :param pair: Pair to get the data for + :return: Ticker dict from exchange or empty dict if ticker is not available for the pair + """ + if self._exchange is None: + raise OperationalException('Exchange is not available to DataProvider.') + try: + return self._exchange.fetch_ticker(pair) + except ExchangeError: + return {} + + def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: + """ + Fetch latest l2 orderbook data + Warning: Does a network request - so use with common sense. + :param pair: pair to get the data for + :param maximum: Maximum number of orderbook entries to query + :return: dict including bids/asks with a total of `maximum` entries. + """ + if self._exchange is None: + raise OperationalException('Exchange is not available to DataProvider.') + return self._exchange.fetch_l2_order_book(pair, maximum) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6d8b329bb..d3e0afe7b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -63,9 +63,7 @@ class Backtesting: self.all_results: Dict[str, Dict] = {} self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - self.dataprovider = DataProvider(self.config, self.exchange) - IStrategy.dp = self.dataprovider if self.config.get('strategy_list', None): for strat in list(self.config['strategy_list']): @@ -132,6 +130,7 @@ class Backtesting: Load strategy into backtesting """ self.strategy: IStrategy = strategy + strategy.dp = self.dataprovider # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case @@ -353,7 +352,6 @@ class Backtesting: # Update dataprovider cache for pair, dataframe in processed.items(): self.dataprovider._set_cached_df(pair, self.timeframe, dataframe) - self.strategy.dp = self.dataprovider # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) From 4b6cd69c81e1a3442941ec03c57f26260e57812d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 May 2021 20:08:31 +0200 Subject: [PATCH 08/16] Add test for no-exchange dataprovider --- freqtrade/data/dataprovider.py | 14 +++++++++----- freqtrade/optimize/hyperopt.py | 1 - tests/data/test_dataprovider.py | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 731815572..aad50e404 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -19,6 +19,8 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) +NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.' + class DataProvider: @@ -126,7 +128,7 @@ class DataProvider: Refresh data, called with each cycle """ if self._exchange is None: - raise OperationalException('Exchange is not available to DataProvider.') + raise OperationalException(NO_EXCHANGE_EXCEPTION) if helping_pairs: self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) else: @@ -139,7 +141,7 @@ class DataProvider: Should be whitelist + open trades. """ if self._exchange is None: - raise OperationalException('Exchange is not available to DataProvider.') + raise OperationalException(NO_EXCHANGE_EXCEPTION) return list(self._exchange._klines.keys()) def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: @@ -151,6 +153,8 @@ class DataProvider: :param copy: copy dataframe before returning if True. Use False only for read-only operations (where the dataframe is not modified) """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): return self._exchange.klines((pair, timeframe or self._config['timeframe']), copy=copy) @@ -164,7 +168,7 @@ class DataProvider: :return: Market data dict from ccxt or None if market info is not available for the pair """ if self._exchange is None: - raise OperationalException('Exchange is not available to DataProvider.') + raise OperationalException(NO_EXCHANGE_EXCEPTION) return self._exchange.markets.get(pair) def ticker(self, pair: str): @@ -174,7 +178,7 @@ class DataProvider: :return: Ticker dict from exchange or empty dict if ticker is not available for the pair """ if self._exchange is None: - raise OperationalException('Exchange is not available to DataProvider.') + raise OperationalException(NO_EXCHANGE_EXCEPTION) try: return self._exchange.fetch_ticker(pair) except ExchangeError: @@ -189,5 +193,5 @@ class DataProvider: :return: dict including bids/asks with a total of `maximum` entries. """ if self._exchange is None: - raise OperationalException('Exchange is not available to DataProvider.') + raise OperationalException(NO_EXCHANGE_EXCEPTION) return self._exchange.fetch_l2_order_book(pair, maximum) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5e3d01047..5ccf02d01 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -31,7 +31,6 @@ from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F4 from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver -from freqtrade.strategy import IStrategy # Suppress scikit-learn FutureWarnings from skopt diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 6b33fa7f2..c3b210d9d 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -246,3 +246,24 @@ def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history): assert dataframe.empty assert isinstance(time, datetime) assert time == datetime(1970, 1, 1, tzinfo=timezone.utc) + + +def test_no_exchange_mode(default_conf): + dp = DataProvider(default_conf, None) + + message = "Exchange is not available to DataProvider." + + with pytest.raises(OperationalException, match=message): + dp.refresh([()]) + + with pytest.raises(OperationalException, match=message): + dp.ohlcv('XRP/USDT', '5m') + + with pytest.raises(OperationalException, match=message): + dp.market('XRP/USDT') + + with pytest.raises(OperationalException, match=message): + dp.ticker('XRP/USDT') + + with pytest.raises(OperationalException, match=message): + dp.orderbook('XRP/USDT', 20) From 1b01ad6f8588d4bd145f05ec6137b30e60b6b352 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Fri, 7 May 2021 17:27:48 +0300 Subject: [PATCH 09/16] Make exchange parameter optional and do not use it as parameter in backtesting. --- freqtrade/data/dataprovider.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index aad50e404..727768ebb 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -24,7 +24,7 @@ NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.' class DataProvider: - def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: + def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None: self._config = config self._exchange = exchange self._pairlists = pairlists diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d3e0afe7b..150baa9bc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -63,7 +63,7 @@ class Backtesting: self.all_results: Dict[str, Dict] = {} self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - self.dataprovider = DataProvider(self.config, self.exchange) + self.dataprovider = DataProvider(self.config, None) if self.config.get('strategy_list', None): for strat in list(self.config['strategy_list']): From f1eb6535452fb1598b9999b59ae2e5167ce074f7 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Fri, 7 May 2021 17:28:06 +0300 Subject: [PATCH 10/16] Fix strategy protections not being loaded in backtesting. --- freqtrade/optimize/backtesting.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 150baa9bc..80eb34c30 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -110,8 +110,6 @@ class Backtesting: PairLocks.timeframe = self.config['timeframe'] PairLocks.use_db = False PairLocks.reset_locks() - if self.config.get('enable_protections', False): - self.protections = ProtectionManager(self.config) self.wallets = Wallets(self.config, self.exchange, log=False) @@ -135,6 +133,12 @@ class Backtesting: # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + if self.config.get('enable_protections', False): + conf = self.config + if hasattr(strategy, 'protections'): + conf = deepcopy(conf) + conf['protections'] = strategy.protections + self.protections = ProtectionManager(conf) def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ From 8d8c782bd039780f8563e8edf7f735501c9013ab Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 8 May 2021 16:06:19 +0300 Subject: [PATCH 11/16] Slice dataframe in backtesting, preventing access to rows past current time. --- freqtrade/data/dataprovider.py | 21 ++++++++++++++++++--- freqtrade/optimize/backtesting.py | 8 ++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 727768ebb..a5d059e4a 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -20,6 +20,7 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.' +MAX_DATAFRAME_CANDLES = 1000 class DataProvider: @@ -29,6 +30,14 @@ class DataProvider: self._exchange = exchange self._pairlists = pairlists self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} + self.__slice_index = None + + def _set_dataframe_max_index(self, limit_index: int): + """ + Limit analyzed dataframe to max specified index. + :param limit_index: dataframe index. + """ + self.__slice_index = limit_index def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None: """ @@ -85,10 +94,16 @@ class DataProvider: combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - if (pair, timeframe) in self.__cached_pairs: - return self.__cached_pairs[(pair, timeframe)] + pair_key = (pair, timeframe) + if pair_key in self.__cached_pairs: + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + df, date = self.__cached_pairs[pair_key] + else: + max_index = self.__slice_index + df, date = self.__cached_pairs[pair_key] + df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index] + return df, date else: - return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) @property diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 80eb34c30..f7c984047 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -373,8 +373,9 @@ class Backtesting: open_trade_count_start = open_trade_count for i, pair in enumerate(data): + row_index = indexes[pair] try: - row = data[pair][indexes[pair]] + row = data[pair][row_index] except IndexError: # missing Data for one pair at the end. # Warnings for this are shown during data loading @@ -383,7 +384,10 @@ class Backtesting: # Waits until the time-counter reaches the start of the data for this pair. if row[DATE_IDX] > tmp: continue - indexes[pair] += 1 + + row_index += 1 + self.dataprovider._set_dataframe_max_index(row_index) # noqa + indexes[pair] = row_index # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected From 17b9e898d2a234b9c1861192af21618ca63442a1 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 8 May 2021 16:56:59 +0300 Subject: [PATCH 12/16] Update docs displaying how to get last available and trade-open candles. --- docs/strategy-advanced.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 383b2a1a9..ceadf0ab0 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -265,12 +265,6 @@ class AwesomeStrategy(IStrategy): Imagine you want to use `custom_stoploss()` to use a trailing indicator like e.g. "ATR" -!!! Warning - Only use `dataframe` values up until and including `current_time` value. Reading past - `current_time` you will look into the future, which will produce incorrect backtesting results - and throw an exception in dry/live runs. - see [Common mistakes when developing strategies](strategy-customization.md#common-mistakes-when-developing-strategies) for more info. - !!! Note `dataframe['date']` contains the candle's open date. During dry/live runs `current_time` and `trade.open_date_utc` will not match the candle date precisely and using them directly will throw @@ -290,7 +284,6 @@ class AwesomeStrategy(IStrategy): def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: - # Default return value result = 1 if trade: @@ -298,11 +291,19 @@ class AwesomeStrategy(IStrategy): # be rounded to previous candle to be used as dataframe index. Rounding must also be # applied to `trade.open_date(_utc)` if it is used for `dataframe` indexing. dataframe = self.dp.get_analyzed_dataframe(pair, self.timeframe) - current_time = timeframe_to_prev_date(self.timeframe, current_time) - current_row = dataframe.loc[dataframe['date'] == current_time].squeeze() - if 'atr' in current_row: + current_candle = dataframe.loc[-1].squeeze() + if 'atr' in current_candle: # new stoploss relative to current_rate - new_stoploss = (current_rate - current_row['atr']) / current_rate + new_stoploss = (current_rate - current_candle['atr']) / current_rate + + # Round trade date to it's candle time. + trade_date = timeframe_to_prev_date(trade.open_date_utc) + trade_candle = dataframe.loc[dataframe['date'] == trade_date] + # Just opened trades do not have their candle complete yet therefore trade_candle may be None + if trade_candle is not None: + trade_candle = trade_candle.squeeze() + trade_stoploss = (current_rate - trade_candle['atr']) / current_rate + new_stoploss = max(new_stoploss, trade_stoploss) # turn into relative negative offset required by `custom_stoploss` return implementation result = new_stoploss - 1 From 92186d89a2fcbbb03eaf789ccf52238b9b934936 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 May 2021 09:56:36 +0200 Subject: [PATCH 13/16] Add some changes to strategytemplate --- freqtrade/data/dataprovider.py | 9 ++++++--- freqtrade/optimize/backtesting.py | 2 +- .../templates/subtemplates/strategy_methods_advanced.j2 | 7 +++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index a5d059e4a..1a86eece5 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -30,7 +30,7 @@ class DataProvider: self._exchange = exchange self._pairlists = pairlists self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} - self.__slice_index = None + self.__slice_index: Optional[int] = None def _set_dataframe_max_index(self, limit_index: int): """ @@ -88,6 +88,8 @@ class DataProvider: def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]: """ + Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry), + and the last 1000 candles (up to the time evaluated at this moment) in all other modes. :param pair: pair to get the data for :param timeframe: timeframe to get data for :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe @@ -99,9 +101,10 @@ class DataProvider: if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): df, date = self.__cached_pairs[pair_key] else: - max_index = self.__slice_index df, date = self.__cached_pairs[pair_key] - df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index] + if self.__slice_index is not None: + max_index = self.__slice_index + df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index] return df, date else: return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f7c984047..e057d8189 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -386,7 +386,7 @@ class Backtesting: continue row_index += 1 - self.dataprovider._set_dataframe_max_index(row_index) # noqa + self.dataprovider._set_dataframe_max_index(row_index) indexes[pair] = row_index # without positionstacking, we can only have one open trade per pair. diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index c69b52cad..2a9ac0690 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -39,7 +39,7 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', return self.stoploss def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, **kwargs) -> bool: + time_in_force: str, current_time: 'datetime', **kwargs) -> bool: """ Called right before placing a buy order. Timing for this function is critical, so avoid doing heavy computations or @@ -54,6 +54,7 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f :param amount: Amount in target (quote) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the buy-order is placed on the exchange. False aborts the process @@ -61,7 +62,8 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f return True def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float, - rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + rate: float, time_in_force: str, sell_reason: str, + current_time: 'datetime', **kwargs) -> bool: """ Called right before placing a regular sell order. Timing for this function is critical, so avoid doing heavy computations or @@ -80,6 +82,7 @@ def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: :param sell_reason: Sell reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', 'sell_signal', 'force_sell', 'emergency_sell'] + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the sell-order is placed on the exchange. False aborts the process From 00e93dad024c2e1be045d4d66cbba3d52928e081 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 May 2021 10:04:56 +0200 Subject: [PATCH 14/16] Fix mistake in the docs --- docs/strategy-advanced.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index ceadf0ab0..e52beec3b 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -61,7 +61,7 @@ from freqtrade.strategy import IStrategy, timeframe_to_prev_date class AwesomeStrategy(IStrategy): def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, current_profit: float, **kwargs): - dataframe = self.dp.get_analyzed_dataframe(pair, self.timeframe) + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) trade_open_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc) trade_row = dataframe.loc[dataframe['date'] == trade_open_date].squeeze() @@ -290,7 +290,7 @@ class AwesomeStrategy(IStrategy): # Using current_time directly would only work in backtesting. Live/dry runs need time to # be rounded to previous candle to be used as dataframe index. Rounding must also be # applied to `trade.open_date(_utc)` if it is used for `dataframe` indexing. - dataframe = self.dp.get_analyzed_dataframe(pair, self.timeframe) + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) current_candle = dataframe.loc[-1].squeeze() if 'atr' in current_candle: # new stoploss relative to current_rate From 1c408c04041830cac86969bec0d026afa9717870 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 May 2021 19:47:37 +0200 Subject: [PATCH 15/16] Add small tests for backtest mode --- tests/data/test_dataprovider.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index c3b210d9d..b87258c64 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -247,6 +247,25 @@ def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history): assert isinstance(time, datetime) assert time == datetime(1970, 1, 1, tzinfo=timezone.utc) + # Test backtest mode + default_conf["runmode"] = RunMode.BACKTEST + dp._set_dataframe_max_index(1) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + + assert len(dataframe) == 1 + + dp._set_dataframe_max_index(2) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == 2 + + dp._set_dataframe_max_index(3) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == 3 + + dp._set_dataframe_max_index(500) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == len(ohlcv_history) + def test_no_exchange_mode(default_conf): dp = DataProvider(default_conf, None) @@ -267,3 +286,6 @@ def test_no_exchange_mode(default_conf): with pytest.raises(OperationalException, match=message): dp.orderbook('XRP/USDT', 20) + + with pytest.raises(OperationalException, match=message): + dp.available_pairs() From d495ea36932011babb3dcafeaa1795a5e986f71c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 May 2021 19:53:41 +0200 Subject: [PATCH 16/16] Update docs about availability of get_analyzed --- docs/strategy-customization.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 6c62c1e86..49456c047 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -422,10 +422,6 @@ if self.dp: Returns an empty dataframe if the requested pair was not cached. This should not happen when using whitelisted pairs. - -!!! Warning "Warning about backtesting" - This method will return an empty dataframe during backtesting. - ### *orderbook(pair, maximum)* ``` python