diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index eb322df9d..e52beec3b 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -60,7 +60,8 @@ from freqtrade.strategy import IStrategy, timeframe_to_prev_date class AwesomeStrategy(IStrategy): def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, - current_profit: float, dataframe: DataFrame, **kwargs): + current_profit: float, **kwargs): + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) trade_open_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc) trade_row = dataframe.loc[dataframe['date'] == trade_open_date].squeeze() @@ -105,8 +106,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -156,8 +156,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. if current_time - timedelta(minutes=120) > trade.open_date_utc: @@ -183,8 +182,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: if pair in ('ETH/BTC', 'XRP/BTC'): return -0.10 @@ -210,8 +208,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: if current_profit < 0.04: return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss @@ -250,8 +247,7 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: # evaluate highest to lowest, so that highest possible stop is used if current_profit > 0.40: @@ -269,12 +265,6 @@ class AwesomeStrategy(IStrategy): Imagine you want to use `custom_stoploss()` to use a trailing indicator like e.g. "ATR" -!!! Warning - Only use `dataframe` values up until and including `current_time` value. Reading past - `current_time` you will look into the future, which will produce incorrect backtesting results - and throw an exception in dry/live runs. - see [Common mistakes when developing strategies](strategy-customization.md#common-mistakes-when-developing-strategies) for more info. - !!! Note `dataframe['date']` contains the candle's open date. During dry/live runs `current_time` and `trade.open_date_utc` will not match the candle date precisely and using them directly will throw @@ -293,20 +283,27 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: - + current_rate: float, current_profit: float, **kwargs) -> float: # Default return value result = 1 if trade: # Using current_time directly would only work in backtesting. Live/dry runs need time to # be rounded to previous candle to be used as dataframe index. Rounding must also be # applied to `trade.open_date(_utc)` if it is used for `dataframe` indexing. - current_time = timeframe_to_prev_date(self.timeframe, current_time) - current_row = dataframe.loc[dataframe['date'] == current_time].squeeze() - if 'atr' in current_row: + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + current_candle = dataframe.loc[-1].squeeze() + if 'atr' in current_candle: # new stoploss relative to current_rate - new_stoploss = (current_rate - current_row['atr']) / current_rate + new_stoploss = (current_rate - current_candle['atr']) / current_rate + + # Round trade date to it's candle time. + trade_date = timeframe_to_prev_date(trade.open_date_utc) + trade_candle = dataframe.loc[dataframe['date'] == trade_date] + # Just opened trades do not have their candle complete yet therefore trade_candle may be None + if trade_candle is not None: + trade_candle = trade_candle.squeeze() + trade_stoploss = (current_rate - trade_candle['atr']) / current_rate + new_stoploss = max(new_stoploss, trade_stoploss) # turn into relative negative offset required by `custom_stoploss` return implementation result = new_stoploss - 1 diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 59bfbde48..49456c047 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -422,10 +422,6 @@ if self.dp: Returns an empty dataframe if the requested pair was not cached. This should not happen when using whitelisted pairs. - -!!! Warning "Warning about backtesting" - This method will return an empty dataframe during backtesting. - ### *orderbook(pair, maximum)* ``` python @@ -631,8 +627,7 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: # once the profit has risen above 10%, keep the stoploss at 7% above the open price if current_profit > 0.10: diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b4dea0743..1a86eece5 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -19,14 +19,25 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) +NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.' +MAX_DATAFRAME_CANDLES = 1000 + class DataProvider: - def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: + def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None: self._config = config self._exchange = exchange self._pairlists = pairlists self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} + self.__slice_index: Optional[int] = None + + def _set_dataframe_max_index(self, limit_index: int): + """ + Limit analyzed dataframe to max specified index. + :param limit_index: dataframe index. + """ + self.__slice_index = limit_index def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None: """ @@ -45,40 +56,6 @@ class DataProvider: """ self._pairlists = pairlists - def refresh(self, - pairlist: ListPairsWithTimeframes, - helping_pairs: ListPairsWithTimeframes = None) -> None: - """ - Refresh data, called with each cycle - """ - if helping_pairs: - self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) - else: - self._exchange.refresh_latest_ohlcv(pairlist) - - @property - def available_pairs(self) -> ListPairsWithTimeframes: - """ - Return a list of tuples containing (pair, timeframe) for which data is currently cached. - Should be whitelist + open trades. - """ - return list(self._exchange._klines.keys()) - - def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: - """ - Get candle (OHLCV) data for the given pair as DataFrame - Please use the `available_pairs` method to verify which pairs are currently cached. - :param pair: pair to get the data for - :param timeframe: Timeframe to get data for - :param copy: copy dataframe before returning if True. - Use False only for read-only operations (where the dataframe is not modified) - """ - if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines((pair, timeframe or self._config['timeframe']), - copy=copy) - else: - return DataFrame() - def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame: """ Get stored historical candle (OHLCV) data @@ -111,47 +88,27 @@ class DataProvider: def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]: """ + Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry), + and the last 1000 candles (up to the time evaluated at this moment) in all other modes. :param pair: pair to get the data for :param timeframe: timeframe to get data for :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - if (pair, timeframe) in self.__cached_pairs: - return self.__cached_pairs[(pair, timeframe)] + pair_key = (pair, timeframe) + if pair_key in self.__cached_pairs: + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + df, date = self.__cached_pairs[pair_key] + else: + df, date = self.__cached_pairs[pair_key] + if self.__slice_index is not None: + max_index = self.__slice_index + df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index] + return df, date else: - return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) - def market(self, pair: str) -> Optional[Dict[str, Any]]: - """ - Return market data for the pair - :param pair: Pair to get the data for - :return: Market data dict from ccxt or None if market info is not available for the pair - """ - return self._exchange.markets.get(pair) - - def ticker(self, pair: str): - """ - Return last ticker data from exchange - :param pair: Pair to get the data for - :return: Ticker dict from exchange or empty dict if ticker is not available for the pair - """ - try: - return self._exchange.fetch_ticker(pair) - except ExchangeError: - return {} - - def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: - """ - Fetch latest l2 orderbook data - Warning: Does a network request - so use with common sense. - :param pair: pair to get the data for - :param maximum: Maximum number of orderbook entries to query - :return: dict including bids/asks with a total of `maximum` entries. - """ - return self._exchange.fetch_l2_order_book(pair, maximum) - @property def runmode(self) -> RunMode: """ @@ -173,3 +130,86 @@ class DataProvider: return self._pairlists.whitelist.copy() else: raise OperationalException("Dataprovider was not initialized with a pairlist provider.") + + def clear_cache(self): + """ + Clear pair dataframe cache. + """ + self.__cached_pairs = {} + + # Exchange functions + + def refresh(self, + pairlist: ListPairsWithTimeframes, + helping_pairs: ListPairsWithTimeframes = None) -> None: + """ + Refresh data, called with each cycle + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + if helping_pairs: + self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) + else: + self._exchange.refresh_latest_ohlcv(pairlist) + + @property + def available_pairs(self) -> ListPairsWithTimeframes: + """ + Return a list of tuples containing (pair, timeframe) for which data is currently cached. + Should be whitelist + open trades. + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + return list(self._exchange._klines.keys()) + + def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: + """ + Get candle (OHLCV) data for the given pair as DataFrame + Please use the `available_pairs` method to verify which pairs are currently cached. + :param pair: pair to get the data for + :param timeframe: Timeframe to get data for + :param copy: copy dataframe before returning if True. + Use False only for read-only operations (where the dataframe is not modified) + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + return self._exchange.klines((pair, timeframe or self._config['timeframe']), + copy=copy) + else: + return DataFrame() + + def market(self, pair: str) -> Optional[Dict[str, Any]]: + """ + Return market data for the pair + :param pair: Pair to get the data for + :return: Market data dict from ccxt or None if market info is not available for the pair + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + return self._exchange.markets.get(pair) + + def ticker(self, pair: str): + """ + Return last ticker data from exchange + :param pair: Pair to get the data for + :return: Ticker dict from exchange or empty dict if ticker is not available for the pair + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + try: + return self._exchange.fetch_ticker(pair) + except ExchangeError: + return {} + + def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: + """ + Fetch latest l2 orderbook data + Warning: Does a network request - so use with common sense. + :param pair: pair to get the data for + :param maximum: Maximum number of orderbook entries to query + :return: dict including bids/asks with a total of `maximum` entries. + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + return self._exchange.fetch_l2_order_book(pair, maximum) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c2b15d23f..b3379a462 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,7 +11,6 @@ from typing import Any, Dict, List, Optional import arrow from cachetools import TTLCache -from pandas import DataFrame from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency @@ -553,7 +552,7 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, - time_in_force=time_in_force): + time_in_force=time_in_force, current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of buying {pair}") return False amount = self.exchange.amount_to_precision(pair, amount) @@ -784,10 +783,10 @@ class FreqtradeBot(LoggingMixin): config_ask_strategy = self.config.get('ask_strategy', {}) - analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, - self.strategy.timeframe) if (config_ask_strategy.get('use_sell_signal', True) or config_ask_strategy.get('ignore_roi_if_buy_signal', False)): + analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, + self.strategy.timeframe) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df) @@ -814,13 +813,13 @@ class FreqtradeBot(LoggingMixin): # resulting in outdated RPC messages self._sell_rate_cache[trade.pair] = sell_rate - if self._check_and_execute_sell(analyzed_df, trade, sell_rate, buy, sell): + if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True else: logger.debug('checking sell') sell_rate = self.get_sell_rate(trade.pair, True) - if self._check_and_execute_sell(analyzed_df, trade, sell_rate, buy, sell): + if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True logger.debug('Found no sell signal for %s.', trade) @@ -951,13 +950,13 @@ class FreqtradeBot(LoggingMixin): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") - def _check_and_execute_sell(self, dataframe: DataFrame, trade: Trade, sell_rate: float, + def _check_and_execute_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: """ Check and execute sell """ should_sell = self.strategy.should_sell( - dataframe, trade, sell_rate, datetime.now(timezone.utc), buy, sell, + trade, sell_rate, datetime.now(timezone.utc), buy, sell, force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) @@ -1191,8 +1190,8 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, - time_in_force=time_in_force, - sell_reason=sell_reason.sell_reason): + time_in_force=time_in_force, sell_reason=sell_reason.sell_reason, + current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of selling {trade.pair}") return False diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 899da03e4..e057d8189 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -63,9 +63,7 @@ class Backtesting: self.all_results: Dict[str, Dict] = {} self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - - dataprovider = DataProvider(self.config, self.exchange) - IStrategy.dp = dataprovider + self.dataprovider = DataProvider(self.config, None) if self.config.get('strategy_list', None): for strat in list(self.config['strategy_list']): @@ -96,7 +94,7 @@ class Backtesting: "PrecisionFilter not allowed for backtesting multiple strategies." ) - dataprovider.add_pairlisthandler(self.pairlists) + self.dataprovider.add_pairlisthandler(self.pairlists) self.pairlists.refresh_pairlist() if len(self.pairlists.whitelist) == 0: @@ -112,8 +110,6 @@ class Backtesting: PairLocks.timeframe = self.config['timeframe'] PairLocks.use_db = False PairLocks.reset_locks() - if self.config.get('enable_protections', False): - self.protections = ProtectionManager(self.config) self.wallets = Wallets(self.config, self.exchange, log=False) @@ -132,10 +128,17 @@ class Backtesting: Load strategy into backtesting """ self.strategy: IStrategy = strategy + strategy.dp = self.dataprovider # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + if self.config.get('enable_protections', False): + conf = self.config + if hasattr(strategy, 'protections'): + conf = deepcopy(conf) + conf['protections'] = strategy.protections + self.protections = ProtectionManager(conf) def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ @@ -176,6 +179,7 @@ class Backtesting: Trade.use_db = False PairLocks.reset_locks() Trade.reset_trades() + self.dataprovider.clear_cache() def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]: """ @@ -247,10 +251,9 @@ class Backtesting: else: return sell_row[OPEN_IDX] - def _get_sell_trade_entry(self, dataframe: DataFrame, trade: LocalTrade, - sell_row: Tuple) -> Optional[LocalTrade]: + def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: - sell = self.strategy.should_sell(dataframe, trade, sell_row[OPEN_IDX], # type: ignore + sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX], sell_row[SELL_IDX], low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX]) @@ -267,7 +270,8 @@ class Backtesting: pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, rate=closerate, time_in_force=time_in_force, - sell_reason=sell.sell_reason): + sell_reason=sell.sell_reason, + current_time=sell_row[DATE_IDX].to_pydatetime()): return None trade.close(closerate, show_msg=False) @@ -287,7 +291,7 @@ class Backtesting: # Confirm trade entry: if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX], - time_in_force=time_in_force): + time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()): return None if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): @@ -349,6 +353,10 @@ class Backtesting: trades: List[LocalTrade] = [] self.prepare_backtest(enable_protections) + # Update dataprovider cache + for pair, dataframe in processed.items(): + self.dataprovider._set_cached_df(pair, self.timeframe, dataframe) + # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) data: Dict = self._get_ohlcv_as_lists(processed) @@ -365,8 +373,9 @@ class Backtesting: open_trade_count_start = open_trade_count for i, pair in enumerate(data): + row_index = indexes[pair] try: - row = data[pair][indexes[pair]] + row = data[pair][row_index] except IndexError: # missing Data for one pair at the end. # Warnings for this are shown during data loading @@ -375,7 +384,10 @@ class Backtesting: # Waits until the time-counter reaches the start of the data for this pair. if row[DATE_IDX] > tmp: continue - indexes[pair] += 1 + + row_index += 1 + self.dataprovider._set_dataframe_max_index(row_index) + indexes[pair] = row_index # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected @@ -398,7 +410,7 @@ class Backtesting: for trade in open_trades[pair]: # also check the buying candle for sell conditions. - trade_entry = self._get_sell_trade_entry(processed[pair], trade, row) + trade_entry = self._get_sell_trade_entry(trade, row) # Sell occured if trade_entry: # logger.debug(f"{pair} - Backtesting sell {trade}") diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e0a6d50a0..5ccf02d01 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -31,7 +31,6 @@ from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F4 from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver -from freqtrade.strategy import IStrategy # Suppress scikit-learn FutureWarnings from skopt @@ -372,8 +371,6 @@ class Hyperopt: self.backtesting.exchange._api_async = None # type: ignore # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore - self.backtesting.strategy.dp = None # type: ignore - IStrategy.dp = None # type: ignore cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 63dcc75d9..7483abf6d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -229,7 +229,7 @@ class IStrategy(ABC, HyperStrategyMixin): pass def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, **kwargs) -> bool: + time_in_force: str, current_time: datetime, **kwargs) -> bool: """ Called right before placing a buy order. Timing for this function is critical, so avoid doing heavy computations or @@ -244,6 +244,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param amount: Amount in target (quote) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the buy-order is placed on the exchange. False aborts the process @@ -251,7 +252,8 @@ class IStrategy(ABC, HyperStrategyMixin): return True def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float, - rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + rate: float, time_in_force: str, sell_reason: str, + current_time: datetime, **kwargs) -> bool: """ Called right before placing a regular sell order. Timing for this function is critical, so avoid doing heavy computations or @@ -270,6 +272,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param sell_reason: Sell reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', 'sell_signal', 'force_sell', 'emergency_sell'] + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the sell-order is placed on the exchange. False aborts the process @@ -277,7 +280,7 @@ class IStrategy(ABC, HyperStrategyMixin): return True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, dataframe: DataFrame, **kwargs) -> float: + current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -293,15 +296,13 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. - :param dataframe: Analyzed dataframe for this pair. Can contain future data in backtesting. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New stoploss value, relative to the currentrate """ return self.stoploss def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, dataframe: DataFrame, - **kwargs) -> Optional[Union[str, bool]]: + current_profit: float, **kwargs) -> Optional[Union[str, bool]]: """ Custom sell signal logic indicating that specified position should be sold. Returning a string or True from this method is equal to setting sell signal on a candle at specified @@ -539,8 +540,8 @@ class IStrategy(ABC, HyperStrategyMixin): else: return False - def should_sell(self, dataframe: DataFrame, trade: Trade, rate: float, date: datetime, - buy: bool, sell: bool, low: float = None, high: float = None, + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, + sell: bool, low: float = None, high: float = None, force_stoploss: float = 0) -> SellCheckTuple: """ This function evaluates if one of the conditions required to trigger a sell @@ -556,9 +557,8 @@ class IStrategy(ABC, HyperStrategyMixin): trade.adjust_min_max_rates(high or current_rate) - stoplossflag = self.stop_loss_reached(dataframe=dataframe, current_rate=current_rate, - trade=trade, current_time=date, - current_profit=current_profit, + stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, + current_time=date, current_profit=current_profit, force_stoploss=force_stoploss, high=high) # Set current rate to high for backtesting sell @@ -583,7 +583,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: custom_reason = strategy_safe_wrapper(self.custom_sell, default_retval=False)( pair=trade.pair, trade=trade, current_time=date, current_rate=current_rate, - current_profit=current_profit, dataframe=dataframe) + current_profit=current_profit) if custom_reason: sell_signal = SellType.CUSTOM_SELL if isinstance(custom_reason, str): @@ -620,7 +620,7 @@ class IStrategy(ABC, HyperStrategyMixin): # logger.debug(f"{trade.pair} - No sell signal.") return SellCheckTuple(sell_type=SellType.NONE) - def stop_loss_reached(self, dataframe: DataFrame, current_rate: float, trade: Trade, + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, force_stoploss: float, high: float = None) -> SellCheckTuple: """ @@ -638,8 +638,7 @@ class IStrategy(ABC, HyperStrategyMixin): )(pair=trade.pair, trade=trade, current_time=current_time, current_rate=current_rate, - current_profit=current_profit, - dataframe=dataframe) + current_profit=current_profit) # Sanity check - error cases will return None if stop_loss_value: # logger.info(f"{trade.pair} {stop_loss_value=} {current_profit=}") diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index c69b52cad..2a9ac0690 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -39,7 +39,7 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', return self.stoploss def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, **kwargs) -> bool: + time_in_force: str, current_time: 'datetime', **kwargs) -> bool: """ Called right before placing a buy order. Timing for this function is critical, so avoid doing heavy computations or @@ -54,6 +54,7 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f :param amount: Amount in target (quote) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the buy-order is placed on the exchange. False aborts the process @@ -61,7 +62,8 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f return True def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float, - rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + rate: float, time_in_force: str, sell_reason: str, + current_time: 'datetime', **kwargs) -> bool: """ Called right before placing a regular sell order. Timing for this function is critical, so avoid doing heavy computations or @@ -80,6 +82,7 @@ def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: :param sell_reason: Sell reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', 'sell_signal', 'force_sell', 'emergency_sell'] + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the sell-order is placed on the exchange. False aborts the process diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 6b33fa7f2..b87258c64 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -246,3 +246,46 @@ def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history): assert dataframe.empty assert isinstance(time, datetime) assert time == datetime(1970, 1, 1, tzinfo=timezone.utc) + + # Test backtest mode + default_conf["runmode"] = RunMode.BACKTEST + dp._set_dataframe_max_index(1) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + + assert len(dataframe) == 1 + + dp._set_dataframe_max_index(2) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == 2 + + dp._set_dataframe_max_index(3) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == 3 + + dp._set_dataframe_max_index(500) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == len(ohlcv_history) + + +def test_no_exchange_mode(default_conf): + dp = DataProvider(default_conf, None) + + message = "Exchange is not available to DataProvider." + + with pytest.raises(OperationalException, match=message): + dp.refresh([()]) + + with pytest.raises(OperationalException, match=message): + dp.ohlcv('XRP/USDT', '5m') + + with pytest.raises(OperationalException, match=message): + dp.market('XRP/USDT') + + with pytest.raises(OperationalException, match=message): + dp.ticker('XRP/USDT') + + with pytest.raises(OperationalException, match=message): + dp.orderbook('XRP/USDT', 20) + + with pytest.raises(OperationalException, match=message): + dp.available_pairs() diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index a8862e9c9..92ac9f63a 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -36,10 +36,11 @@ def test_default_strategy(result, fee): ) assert strategy.confirm_trade_entry(pair='ETH/BTC', order_type='limit', amount=0.1, - rate=20000, time_in_force='gtc') is True + rate=20000, time_in_force='gtc', + current_time=datetime.utcnow()) is True assert strategy.confirm_trade_exit(pair='ETH/BTC', trade=trade, order_type='limit', amount=0.1, - rate=20000, time_in_force='gtc', sell_reason='roi') is True + rate=20000, time_in_force='gtc', sell_reason='roi', + current_time=datetime.utcnow()) is True assert strategy.custom_stoploss(pair='ETH/BTC', trade=trade, current_time=datetime.now(), - current_rate=20_000, current_profit=0.05, dataframe=None - ) == strategy.stoploss + current_rate=20_000, current_profit=0.05) == strategy.stoploss diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a241d7f43..ded396779 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -360,7 +360,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili now = arrow.utcnow().datetime sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit), trade=trade, current_time=now, current_profit=profit, - force_stoploss=0, high=None, dataframe=None) + force_stoploss=0, high=None) assert isinstance(sl_flag, SellCheckTuple) assert sl_flag.sell_type == expected if expected == SellType.NONE: @@ -371,7 +371,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit2), trade=trade, current_time=now, current_profit=profit2, - force_stoploss=0, high=None, dataframe=None) + force_stoploss=0, high=None) assert sl_flag.sell_type == expected2 if expected2 == SellType.NONE: assert sl_flag.sell_flag is False @@ -399,27 +399,27 @@ def test_custom_sell(default_conf, fee, caplog) -> None: ) now = arrow.utcnow().datetime - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_flag is False assert res.sell_type == SellType.NONE strategy.custom_sell = MagicMock(return_value=True) - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_flag is True assert res.sell_type == SellType.CUSTOM_SELL assert res.sell_reason == 'custom_sell' strategy.custom_sell = MagicMock(return_value='hello world') - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_type == SellType.CUSTOM_SELL assert res.sell_flag is True assert res.sell_reason == 'hello world' caplog.clear() strategy.custom_sell = MagicMock(return_value='h' * 100) - res = strategy.should_sell(None, trade, 1, now, False, False, None, None, 0) + res = strategy.should_sell(trade, 1, now, False, False, None, None, 0) assert res.sell_type == SellType.CUSTOM_SELL assert res.sell_flag is True assert res.sell_reason == 'h' * 64