adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3497 from freqtrade/keep_dataframe_noapi

Browse Source 
Analyze dataframe and keep it until the next analysis
This commit is contained in:
hroff-1902
2020-07-05 13:46:02 +03:00
committed by GitHub
parent 5a189ae202 75318525a9
commit 839b3340e6
16 changed files with 603 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
freqtrade/constants.py
View File

@@ -339,4 +339,5 @@ CANCEL_REASON = {
}
# List of pairs with their timeframes
ListPairsWithTimeframes = List[Tuple[str, str]]
PairWithTimeframe = Tuple[str, str]
ListPairsWithTimeframes = List[PairWithTimeframe]

33
freqtrade/data/dataprovider.py
View File

@@ -5,16 +5,17 @@ including ticker and orderbook data, live and historical candle (OHLCV) data
Common Interface for bot and strategy to access data.
"""
import logging
from typing import Any, Dict, List, Optional
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
from arrow import Arrow
from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
from freqtrade.data.history import load_pair_history
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange
from freqtrade.state import RunMode
from freqtrade.constants import ListPairsWithTimeframes
logger = logging.getLogger(__name__)
@@ -25,6 +26,18 @@ class DataProvider:
self._config = config
self._exchange = exchange
self._pairlists = pairlists
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None:
"""
Store cached Dataframe.
Using private method as this should never be used by a user
(but the class is exposed via `self.dp` to the strategy)
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param dataframe: analyzed dataframe
"""
self.__cached_pairs[(pair, timeframe)] = (dataframe, Arrow.utcnow().datetime)
def refresh(self,
pairlist: ListPairsWithTimeframes,
@@ -89,6 +102,20 @@ class DataProvider:
logger.warning(f"No data found for ({pair}, {timeframe}).")
return data
def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]:
"""
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
:return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe
combination.
Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached.
"""
if (pair, timeframe) in self.__cached_pairs:
return self.__cached_pairs[(pair, timeframe)]
else:
return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
def market(self, pair: str) -> Optional[Dict[str, Any]]:
"""
Return market data for the pair

32
freqtrade/freqtradebot.py
View File

@@ -153,6 +153,10 @@ class FreqtradeBot:
self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist),
self.strategy.informative_pairs())
strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()
self.strategy.analyze(self.active_pair_whitelist)
with self._sell_lock:
# Check and handle any timed out open orders
self.check_handle_timedout()
@@ -440,9 +444,8 @@ class FreqtradeBot:
return False
# running get_signal on historical data fetched
(buy, sell) = self.strategy.get_signal(
pair, self.strategy.timeframe,
self.dataprovider.ohlcv(pair, self.strategy.timeframe))
analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe)
(buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
if buy and not sell:
stake_amount = self.get_trade_stake_amount(pair)
@@ -515,6 +518,12 @@ class FreqtradeBot:
amount = stake_amount / buy_limit_requested
order_type = self.strategy.order_types['buy']
if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force):
logger.info(f"User requested abortion of buying {pair}")
return False
order = self.exchange.buy(pair=pair, ordertype=order_type,
amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force)
@@ -717,9 +726,10 @@ class FreqtradeBot:
if (config_ask_strategy.get('use_sell_signal', True) or
config_ask_strategy.get('ignore_roi_if_buy_signal', False)):
(buy, sell) = self.strategy.get_signal(
trade.pair, self.strategy.timeframe,
self.dataprovider.ohlcv(trade.pair, self.strategy.timeframe))
analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair,
self.strategy.timeframe)
(buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df)
if config_ask_strategy.get('use_order_book', False):
order_book_min = config_ask_strategy.get('order_book_min', 1)
@@ -1097,12 +1107,20 @@ class FreqtradeBot:
order_type = self.strategy.order_types.get("emergencysell", "market")
amount = self._safe_sell_amount(trade.pair, trade.amount)
time_in_force = self.strategy.order_time_in_force['sell']
if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit,
time_in_force=time_in_force,
sell_reason=sell_reason.value):
logger.info(f"User requested abortion of selling {trade.pair}")
return False
# Execute sell and update trade record
order = self.exchange.sell(pair=str(trade.pair),
ordertype=order_type,
amount=amount, rate=limit,
time_in_force=self.strategy.order_time_in_force['sell']
time_in_force=time_in_force
)
trade.open_order_id = order['id']

157
freqtrade/strategy/interface.py
View File

@@ -7,20 +7,19 @@ import warnings
from abc import ABC, abstractmethod
from datetime import datetime, timezone
from enum import Enum
from typing import Dict, NamedTuple, Optional, Tuple
from typing import Dict, List, NamedTuple, Optional, Tuple
import arrow
from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import StrategyError
from freqtrade.exceptions import StrategyError, OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.persistence import Trade
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.wallets import Wallets
logger = logging.getLogger(__name__)
@@ -191,6 +190,63 @@ class IStrategy(ABC):
"""
return False
def bot_loop_start(self, **kwargs) -> None:
"""
Called at the start of the bot iteration (one loop).
Might be used to perform pair-independent tasks
(e.g. gather some remote resource for comparison)
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
"""
pass
def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
time_in_force: str, **kwargs) -> bool:
"""
Called right before placing a buy order.
Timing for this function is critical, so avoid doing heavy computations or
network requests in this method.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns True (always confirming).
:param pair: Pair that's about to be bought.
:param order_type: Order type (as configured in order_types). usually limit or market.
:param amount: Amount in target (quote) currency that's going to be traded.
:param rate: Rate that's going to be used when using limit orders
:param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return bool: When True is returned, then the buy-order is placed on the exchange.
False aborts the process
"""
return True
def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float,
rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
"""
Called right before placing a regular sell order.
Timing for this function is critical, so avoid doing heavy computations or
network requests in this method.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns True (always confirming).
:param pair: Pair that's about to be sold.
:param trade: trade object.
:param order_type: Order type (as configured in order_types). usually limit or market.
:param amount: Amount in quote currency.
:param rate: Rate that's going to be used when using limit orders
:param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
:param sell_reason: Sell reason.
Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
'sell_signal', 'force_sell', 'emergency_sell']
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return bool: When True is returned, then the sell-order is placed on the exchange.
False aborts the process
"""
return True
def informative_pairs(self) -> ListPairsWithTimeframes:
"""
Define additional, informative pair/interval combinations to be cached from the exchange.
@@ -204,6 +260,10 @@ class IStrategy(ABC):
"""
return []
###
# END - Intended to be overridden by strategy
###
def get_strategy_name(self) -> str:
"""
Returns strategy class name
@@ -273,6 +333,8 @@ class IStrategy(ABC):
# Defs that only make change on new candle data.
dataframe = self.analyze_ticker(dataframe, metadata)
self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date']
if self.dp:
self.dp._set_cached_df(pair, self.timeframe, dataframe)
else:
logger.debug("Skipping TA Analysis for already analyzed candle")
dataframe['buy'] = 0
@@ -284,13 +346,53 @@ class IStrategy(ABC):
return dataframe
def analyze_pair(self, pair: str) -> None:
"""
Fetch data for this pair from dataprovider and analyze.
Stores the dataframe into the dataprovider.
The analyzed dataframe is then accessible via `dp.get_analyzed_dataframe()`.
:param pair: Pair to analyze.
"""
if not self.dp:
raise OperationalException("DataProvider not found.")
dataframe = self.dp.ohlcv(pair, self.timeframe)
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty candle (OHLCV) data for pair %s', pair)
return
try:
df_len, df_close, df_date = self.preserve_df(dataframe)
dataframe = strategy_safe_wrapper(
self._analyze_ticker_internal, message=""
)(dataframe, {'pair': pair})
self.assert_df(dataframe, df_len, df_close, df_date)
except StrategyError as error:
logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}")
return
if dataframe.empty:
logger.warning('Empty dataframe for pair %s', pair)
return
def analyze(self, pairs: List[str]) -> None:
"""
Analyze all pairs using analyze_pair().
:param pairs: List of pairs to analyze
"""
for pair in pairs:
self.analyze_pair(pair)
@staticmethod
def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]:
""" keep some data for dataframes """
return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1]
def assert_df(self, dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime):
""" make sure data is unmodified """
"""
Ensure dataframe (length, last candle) was not modified, and has all elements we need.
"""
message = ""
if df_len != len(dataframe):
message = "length"
@@ -304,31 +406,17 @@ class IStrategy(ABC):
else:
raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.")
def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]:
def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]:
"""
Calculates current signal based several technical analysis indicators
Calculates current signal based based on the buy / sell columns of the dataframe.
Used by Bot to get the signal to buy or sell
:param pair: pair in format ANT/BTC
:param interval: Interval to use (in min)
:param dataframe: Dataframe to analyze
:param timeframe: timeframe to use
:param dataframe: Analyzed dataframe to get signal from.
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty candle (OHLCV) data for pair %s', pair)
return False, False
try:
df_len, df_close, df_date = self.preserve_df(dataframe)
dataframe = strategy_safe_wrapper(
self._analyze_ticker_internal, message=""
)(dataframe, {'pair': pair})
self.assert_df(dataframe, df_len, df_close, df_date)
except StrategyError as error:
logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}")
return False, False
if dataframe.empty:
logger.warning('Empty dataframe for pair %s', pair)
logger.warning(f'Empty candle (OHLCV) data for pair {pair}')
return False, False
latest_date = dataframe['date'].max()
@@ -337,24 +425,18 @@ class IStrategy(ABC):
latest_date = arrow.get(latest_date)
# Check if dataframe is out of date
interval_minutes = timeframe_to_minutes(interval)
timeframe_minutes = timeframe_to_minutes(timeframe)
offset = self.config.get('exchange', {}).get('outdated_offset', 5)
if latest_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))):
if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))):
logger.warning(
'Outdated history for pair %s. Last tick is %s minutes old',
pair,
(arrow.utcnow() - latest_date).seconds // 60
pair, (arrow.utcnow() - latest_date).seconds // 60
)
return False, False
(buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
logger.debug(
'trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'],
pair,
str(buy),
str(sell)
)
logger.debug('trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'], pair, str(buy), str(sell))
return buy, sell
def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool,
@@ -500,7 +582,8 @@ class IStrategy(ABC):
def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
"""
Creates a dataframe and populates indicators for given candle (OHLCV) data
Populates indicators for given candle (OHLCV) data (for multiple pairs)
Does not run advice_buy or advise_sell!
Used by optimize operations only, not during dry / live runs.
Using .copy() to get a fresh copy of the dataframe for every strategy run.
Has positive effects on memory usage for whatever reason - also when

6
freqtrade/strategy/strategy_wrapper.py
View File

@@ -5,7 +5,7 @@ from freqtrade.exceptions import StrategyError
logger = logging.getLogger(__name__)
def strategy_safe_wrapper(f, message: str = "", default_retval=None):
def strategy_safe_wrapper(f, message: str = "", default_retval=None, supress_error=False):
"""
Wrapper around user-provided methods and functions.
Caches all exceptions and returns either the default_retval (if it's not None) or raises
@@ -20,7 +20,7 @@ def strategy_safe_wrapper(f, message: str = "", default_retval=None):
f"Strategy caused the following exception: {error}"
f"{f}"
)
if default_retval is None:
if default_retval is None and not supress_error:
raise StrategyError(str(error)) from error
return default_retval
except Exception as error:
@@ -28,7 +28,7 @@ def strategy_safe_wrapper(f, message: str = "", default_retval=None):
f"{message}"
f"Unexpected error {error} calling {f}"
)
if default_retval is None:
if default_retval is None and not supress_error:
raise StrategyError(str(error)) from error
return default_retval

61
freqtrade/templates/subtemplates/strategy_methods_advanced.j2
View File

@@ -1,4 +1,65 @@
def bot_loop_start(self, **kwargs) -> None:
"""
Called at the start of the bot iteration (one loop).
Might be used to perform pair-independent tasks
(e.g. gather some remote ressource for comparison)
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, this simply does nothing.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
"""
pass
def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
time_in_force: str, **kwargs) -> bool:
"""
Called right before placing a buy order.
Timing for this function is critical, so avoid doing heavy computations or
network requests in this method.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns True (always confirming).
:param pair: Pair that's about to be bought.
:param order_type: Order type (as configured in order_types). usually limit or market.
:param amount: Amount in target (quote) currency that's going to be traded.
:param rate: Rate that's going to be used when using limit orders
:param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return bool: When True is returned, then the buy-order is placed on the exchange.
False aborts the process
"""
return True
def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float,
rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
"""
Called right before placing a regular sell order.
Timing for this function is critical, so avoid doing heavy computations or
network requests in this method.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns True (always confirming).
:param pair: Pair that's about to be sold.
:param trade: trade object.
:param order_type: Order type (as configured in order_types). usually limit or market.
:param amount: Amount in quote currency.
:param rate: Rate that's going to be used when using limit orders
:param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
:param sell_reason: Sell reason.
Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
'sell_signal', 'force_sell', 'emergency_sell']
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return bool: When True is returned, then the sell-order is placed on the exchange.
False aborts the process
"""
return True
def check_buy_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool:
"""
Check buy timeout function callback.