diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 1c622737f..8592f6cca 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -47,6 +47,12 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy **For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) file as reference.** +!!! Note: Strategies and Backtesting + To avoid problems and unexpected differences between Backtesting and dry/live modes, please be aware + that during backtesting the full time-interval is passed to the `populate_*()` methods at once. + It is therefore best to use vectorized operations (across the whole dataframe, not loops) and + avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle. + ### Customize Indicators Buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. @@ -250,6 +256,95 @@ class Awesomestrategy(IStrategy): !!! Note: If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. +### Additional data (DataProvider) + +The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. + +!!!Note: + The DataProvier is currently not available during backtesting / hyperopt, but this is planned for the future. + +All methods return `None` in case of failure (do not raise an exception). + +Please always check if the `DataProvider` is available to avoid failures during backtesting. + +#### Possible options for DataProvider + +- `available_pairs` - Property with tuples listing cached pairs with their intervals. (pair, interval) +- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame +- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk +- `runmode` - Property containing the current runmode. + +#### ohlcv / historic_ohlcv + +``` python +if self.dp: + if dp.runmode == 'live': + if ('ETH/BTC', ticker_interval) in self.dp.available_pairs: + data_eth = self.dp.ohlcv(pair='ETH/BTC', + ticker_interval=ticker_interval) + else: + # Get historic ohlcv data (cached on disk). + history_eth = self.dp.historic_ohlcv(pair='ETH/BTC', + ticker_interval='1h') +``` + +!!! Warning: Warning about backtesting + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` provides the full time-range in one go, + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + +#### Available Pairs + +``` python +if self.dp: + for pair, ticker in self.dp.available_pairs: + print(f"available {pair}, {ticker}") +``` + +#### Get data for non-tradeable pairs + +Data for additional, informative pairs (reference pairs) can be beneficial for some strategies. +Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above). +These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting. + +The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument. + +Sample: + +``` python +def informative_pairs(self): + return [("ETH/USDT", "5m"), + ("BTC/TUSD", "15m"), + ] +``` + +!!! Warning: + As these pairs will be refreshed as part of the regular whitelist refresh, it's best to keep this list short. + All intervals and all pairs can be specified as long as they are available (and active) on the used exchange. + It is however better to use resampling to longer time-intervals when possible + to avoid hammering the exchange with too many requests and risk beeing blocked. + +### Additional data - Wallets + +The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. + +!!!NOTE: + Wallets is not available during backtesting / hyperopt. + +Please always check if `Wallets` is available to avoid failures during backtesting. + +``` python +if self.wallets: + free_eth = self.wallets.get_free('ETH') + used_eth = self.wallets.get_used('ETH') + total_eth = self.wallets.get_total('ETH') +``` + +#### Possible options for Wallets + +- `get_free(asset)` - currently available balance to trade +- `get_used(asset)` - currently tied up balance (open orders) +- `get_total(asset)` - total available balance - sum of the 2 above + ### Where is the default strategy? The default buy strategy is located in the file diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 9fd93629f..d972f50b8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,6 +12,7 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -34,9 +35,10 @@ class Configuration(object): Reuse this class for the bot, backtesting, hyperopt and every script that required configuration """ - def __init__(self, args: Namespace) -> None: + def __init__(self, args: Namespace, runmode: RunMode = None) -> None: self.args = args self.config: Optional[Dict[str, Any]] = None + self.runmode = runmode def load_config(self) -> Dict[str, Any]: """ @@ -68,6 +70,13 @@ class Configuration(object): # Load Hyperopt config = self._load_hyperopt_config(config) + # Set runmode + if not self.runmode: + # Handle real mode, infer dry/live from config + self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE + + config.update({'runmode': self.runmode}) + return config def _load_config_file(self, path: str) -> Dict[str, Any]: diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py new file mode 100644 index 000000000..375b8bf5b --- /dev/null +++ b/freqtrade/data/dataprovider.py @@ -0,0 +1,97 @@ +""" +Dataprovider +Responsible to provide data to the bot +including Klines, tickers, historic data +Common Interface for bot and strategy to access data. +""" +import logging +from pathlib import Path +from typing import List, Tuple + +from pandas import DataFrame + +from freqtrade.data.history import load_pair_history +from freqtrade.exchange import Exchange +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +class DataProvider(object): + + def __init__(self, config: dict, exchange: Exchange) -> None: + self._config = config + self._exchange = exchange + + def refresh(self, + pairlist: List[Tuple[str, str]], + helping_pairs: List[Tuple[str, str]] = None) -> None: + """ + Refresh data, called with each cycle + """ + if helping_pairs: + self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) + else: + self._exchange.refresh_latest_ohlcv(pairlist) + + @property + def available_pairs(self) -> List[Tuple[str, str]]: + """ + Return a list of tuples containing pair, tick_interval for which data is currently cached. + Should be whitelist + open trades. + """ + return list(self._exchange._klines.keys()) + + def ohlcv(self, pair: str, tick_interval: str = None, copy: bool = True) -> DataFrame: + """ + get ohlcv data for the given pair as DataFrame + Please check `available_pairs` to verify which pairs are currently cached. + :param pair: pair to get the data for + :param tick_interval: ticker_interval to get pair for + :param copy: copy dataframe before returning. + Use false only for RO operations (where the dataframe is not modified) + """ + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + if tick_interval: + pairtick = (pair, tick_interval) + else: + pairtick = (pair, self._config['ticker_interval']) + + return self._exchange.klines(pairtick, copy=copy) + else: + return DataFrame() + + def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: + """ + get stored historic ohlcv data + :param pair: pair to get the data for + :param tick_interval: ticker_interval to get pair for + """ + return load_pair_history(pair=pair, + ticker_interval=ticker_interval, + refresh_pairs=False, + datadir=Path(self._config['datadir']) if self._config.get( + 'datadir') else None + ) + + def ticker(self, pair: str): + """ + Return last ticker data + """ + # TODO: Implement me + pass + + def orderbook(self, pair: str, max: int): + """ + return latest orderbook data + """ + # TODO: Implement me + pass + + @property + def runmode(self) -> RunMode: + """ + Get runmode of the bot + can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". + """ + return RunMode(self._config.get('runmode', RunMode.OTHER)) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index e0e4d7723..e4d83cf6d 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -80,10 +80,10 @@ class Exchange(object): self._cached_ticker: Dict[str, Any] = {} # Holds last candle refreshed time of each pair - self._pairs_last_refresh_time: Dict[str, int] = {} + self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} # Holds candles - self._klines: Dict[str, DataFrame] = {} + self._klines: Dict[Tuple[str, str], DataFrame] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -158,11 +158,12 @@ class Exchange(object): """exchange ccxt id""" return self._api.id - def klines(self, pair: str, copy=True) -> DataFrame: - if pair in self._klines: - return self._klines[pair].copy() if copy else self._klines[pair] + def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame: + # create key tuple + if pair_interval in self._klines: + return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] else: - return None + return DataFrame() def set_sandbox(self, api, exchange_config: dict, name: str): if exchange_config.get('sandbox'): @@ -518,42 +519,41 @@ class Exchange(object): input_coroutines = [self._async_get_candle_history( pair, tick_interval, since) for since in range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) # Combine tickers data: List = [] - for p, ticker in tickers: + for p, ticker_interval, ticker in tickers: if p == pair: data.extend(ticker) - # Sort data again after extending the result - above calls return in "async order" order + # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) logger.info("downloaded %s with length %s.", pair, len(data)) return data - def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None: + def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: """ - Refresh tickers asyncronously and set `_klines` of this object with the result + Refresh in-memory ohlcv asyncronously and set `_klines` with the result """ - logger.debug("Refreshing klines for %d pairs", len(pair_list)) - asyncio.get_event_loop().run_until_complete( - self.async_get_candles_history(pair_list, ticker_interval)) + logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) - async def async_get_candles_history(self, pairs: List[str], - tick_interval: str) -> List[Tuple[str, List]]: - """Download ohlcv history for pair-list asyncronously """ - # Calculating ticker interval in second - interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 input_coroutines = [] # Gather corotines to run - for pair in pairs: - if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= - arrow.utcnow().timestamp and pair in self._klines): - input_coroutines.append(self._async_get_candle_history(pair, tick_interval)) - else: - logger.debug("Using cached klines data for %s ...", pair) + for pair, ticker_interval in set(pair_list): + # Calculating ticker interval in second + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 - tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + if not ((self._pairs_last_refresh_time.get((pair, ticker_interval), 0) + + interval_in_sec) >= arrow.utcnow().timestamp + and (pair, ticker_interval) in self._klines): + input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) + else: + logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval) + + tickers = asyncio.get_event_loop().run_until_complete( + asyncio.gather(*input_coroutines, return_exceptions=True)) # handle caching for res in tickers: @@ -561,20 +561,26 @@ class Exchange(object): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue pair = res[0] - ticks = res[1] + tick_interval = res[1] + ticks = res[2] # keeping last candle time as last refreshed time of the pair if ticks: - self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 - # keeping parsed dataframe in cache - self._klines[pair] = parse_ticker_dataframe(ticks, tick_interval, fill_missing=True) + self._pairs_last_refresh_time[(pair, tick_interval)] = ticks[-1][0] // 1000 + # keeping parsed dataframe in cache + self._klines[(pair, tick_interval)] = parse_ticker_dataframe( + ticks, tick_interval, fill_missing=True) return tickers @retrier_async async def _async_get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> Tuple[str, List]: + since_ms: Optional[int] = None) -> Tuple[str, str, List]: + """ + Asyncronously gets candle histories using fetch_ohlcv + returns tuple: (pair, tick_interval, ohlcv_list) + """ try: # fetch ohlcv asynchronously - logger.debug("fetching %s since %s ...", pair, since_ms) + logger.debug("fetching %s, %s since %s ...", pair, tick_interval, since_ms) data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) @@ -588,9 +594,9 @@ class Exchange(object): data = sorted(data, key=lambda x: x[0]) except IndexError: logger.exception("Error loading %s. Result was %s.", pair, data) - return pair, [] - logger.debug("done fetching %s ...", pair) - return pair, data + return pair, tick_interval, [] + logger.debug("done fetching %s, %s ...", pair, tick_interval) + return pair, tick_interval, data except ccxt.NotSupported as e: raise OperationalException( diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7d14c734c..656c700ac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -15,6 +15,7 @@ from requests.exceptions import RequestException from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.data.converter import order_book_to_dataframe +from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.exchange import Exchange from freqtrade.persistence import Trade @@ -36,9 +37,9 @@ class FreqtradeBot(object): def __init__(self, config: Dict[str, Any]) -> None: """ - Init all variables and object the bot need to work - :param config: configuration dict, you can use the Configuration.get_config() - method to get the config dict. + Init all variables and objects the bot needs to work + :param config: configuration dict, you can use Configuration.get_config() + to get the config dict. """ logger.info( @@ -54,9 +55,15 @@ class FreqtradeBot(object): self.strategy: IStrategy = StrategyResolver(self.config).strategy self.rpc: RPCManager = RPCManager(self) - self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) + self.dataprovider = DataProvider(self.config, self.exchange) + + # Attach Dataprovider to Strategy baseclass + IStrategy.dp = self.dataprovider + # Attach Wallets to Strategy baseclass + IStrategy.wallets = self.wallets + pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist @@ -151,9 +158,6 @@ class FreqtradeBot(object): self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong - # Should be called before refresh_tickers - # Otherwise it will override cached klines in exchange - # with delta value (klines only from last refresh_pairs) if self.edge: self.edge.calculate() self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) @@ -166,8 +170,12 @@ class FreqtradeBot(object): self.active_pair_whitelist.extend([trade.pair for trade in trades if trade.pair not in self.active_pair_whitelist]) + # Create pair-whitelist tuple with (pair, ticker_interval) + pair_whitelist_tuple = [(pair, self.config['ticker_interval']) + for pair in self.active_pair_whitelist] # Refreshing candles - self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) + self.dataprovider.refresh(pair_whitelist_tuple, + self.strategy.informative_pairs()) # First process current opened trades for trade in trades: @@ -317,7 +325,9 @@ class FreqtradeBot(object): # running get_signal on historical data fetched for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines(_pair)) + (buy, sell) = self.strategy.get_signal( + _pair, interval, self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval)) + if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: @@ -578,8 +588,9 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, - self.exchange.klines(trade.pair)) + (buy, sell) = self.strategy.get_signal( + trade.pair, self.strategy.ticker_interval, + self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval)) config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): diff --git a/freqtrade/main.py b/freqtrade/main.py index f27145b45..75b15915b 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -39,7 +39,7 @@ def main(sysargv: List[str]) -> None: return_code = 1 try: # Load and validate configuration - config = Configuration(args).get_config() + config = Configuration(args, None).get_config() # Init the bot freqtrade = FreqtradeBot(config) @@ -76,7 +76,7 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: freqtrade.cleanup() # Create new instance - freqtrade = FreqtradeBot(Configuration(args).get_config()) + freqtrade = FreqtradeBot(Configuration(args, None).get_config()) freqtrade.rpc.send_msg({ 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': 'config reloaded' diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9ac26cc8b..a8f4e530a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -22,6 +22,7 @@ from freqtrade.data import history from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import StrategyResolver +from freqtrade.state import RunMode from freqtrade.strategy.interface import SellType, IStrategy logger = logging.getLogger(__name__) @@ -374,8 +375,9 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') - self.exchange.refresh_tickers(pairs, self.ticker_interval) - data = self.exchange._klines + self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs]) + data = {key[0]: value for key, value in self.exchange._klines.items()} + else: logger.info('Using local backtesting data (using whitelist in given config) ...') @@ -459,7 +461,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: :param args: Cli args from Arguments() :return: Configuration """ - configuration = Configuration(args) + configuration = Configuration(args, RunMode.BACKTEST) config = configuration.get_config() # Ensure we do not use Exchange credentials diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index fdae47b99..9b628cf2e 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -9,10 +9,11 @@ from typing import Dict, Any from tabulate import tabulate from freqtrade.edge import Edge -from freqtrade.configuration import Configuration from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.resolvers import StrategyResolver +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -83,7 +84,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: :param args: Cli args from Arguments() :return: Configuration """ - configuration = Configuration(args) + configuration = Configuration(args, RunMode.EDGECLI) config = configuration.get_config() # Ensure we do not use Exchange credentials diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 6930bed04..f6d39f11c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -25,6 +25,7 @@ from freqtrade.configuration import Configuration from freqtrade.data.history import load_data from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import Backtesting +from freqtrade.state import RunMode from freqtrade.resolvers import HyperOptResolver logger = logging.getLogger(__name__) @@ -306,7 +307,7 @@ def start(args: Namespace) -> None: # Initialize configuration # Monkey patch the configuration with hyperopt_conf.py - configuration = Configuration(args) + configuration = Configuration(args, RunMode.HYPEROPT) logger.info('Starting freqtrade in Hyperopt mode') config = configuration.load_config() diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 0d1c9f9a0..0f71501ae 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -3,11 +3,11 @@ """ This module load custom strategies """ -import inspect import logging import tempfile from base64 import urlsafe_b64decode from collections import OrderedDict +from inspect import getfullargspec from pathlib import Path from typing import Dict, Optional @@ -126,11 +126,9 @@ class StrategyResolver(IResolver): if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, _path) strategy._populate_fun_len = len( - inspect.getfullargspec(strategy.populate_indicators).args) - strategy._buy_fun_len = len( - inspect.getfullargspec(strategy.populate_buy_trend).args) - strategy._sell_fun_len = len( - inspect.getfullargspec(strategy.populate_sell_trend).args) + getfullargspec(strategy.populate_indicators).args) + strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) + strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) return import_strategy(strategy, config=config) except FileNotFoundError: diff --git a/freqtrade/state.py b/freqtrade/state.py index 42bfb6e41..b69c26cb5 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -3,13 +3,26 @@ """ Bot state constant """ -import enum +from enum import Enum -class State(enum.Enum): +class State(Enum): """ Bot application states """ - RUNNING = 0 - STOPPED = 1 - RELOAD_CONF = 2 + RUNNING = 1 + STOPPED = 2 + RELOAD_CONF = 3 + + +class RunMode(Enum): + """ + Bot running mode (backtest, hyperopt, ...) + can be "live", "dry-run", "backtest", "edgecli", "hyperopt". + """ + LIVE = "live" + DRY_RUN = "dry_run" + BACKTEST = "backtest" + EDGECLI = "edgecli" + HYPEROPT = "hyperopt" + OTHER = "other" # Used for plotting scripts and test diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 085a383db..5c7d50a65 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -42,6 +42,19 @@ class DefaultStrategy(IStrategy): 'sell': 'gtc', } + def informative_pairs(self): + """ + Define additional, informative pair/interval combinations to be cached from the exchange. + These pair/interval combinations are non-tradeable, unless they are part + of the whitelist as well. + For more information, please consult the documentation + :return: List of tuples in the format (pair, interval) + Sample: return [("ETH/USDT", "5m"), + ("BTC/USDT", "15m"), + ] + """ + return [] + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 08a5cf1cd..733651df4 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,7 +13,9 @@ import arrow from pandas import DataFrame from freqtrade import constants +from freqtrade.data.dataprovider import DataProvider from freqtrade.persistence import Trade +from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) @@ -93,12 +95,16 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False - # Dict to determine if analysis is necessary - _last_candle_seen_per_pair: Dict[str, datetime] = {} + # Class level variables (intentional) containing + # the dataprovider (dp) (access to other candles, historic data, ...) + # and wallets - access to the current balance. + dp: DataProvider + wallets: Wallets def __init__(self, config: dict) -> None: self.config = config - self._last_candle_seen_per_pair = {} + # Dict to determine if analysis is necessary + self._last_candle_seen_per_pair: Dict[str, datetime] = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -127,6 +133,19 @@ class IStrategy(ABC): :return: DataFrame with sell column """ + def informative_pairs(self) -> List[Tuple[str, str]]: + """ + Define additional, informative pair/interval combinations to be cached from the exchange. + These pair/interval combinations are non-tradeable, unless they are part + of the whitelist as well. + For more information, please consult the documentation + :return: List of tuples in the format (pair, interval) + Sample: return [("ETH/USDT", "5m"), + ("BTC/USDT", "15m"), + ] + """ + return [] + def get_strategy_name(self) -> str: """ Returns strategy class name diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py new file mode 100644 index 000000000..b17bba273 --- /dev/null +++ b/freqtrade/tests/data/test_dataprovider.py @@ -0,0 +1,92 @@ +from unittest.mock import MagicMock + +from pandas import DataFrame + +from freqtrade.data.dataprovider import DataProvider +from freqtrade.state import RunMode +from freqtrade.tests.conftest import get_patched_exchange + + +def test_ohlcv(mocker, default_conf, ticker_history): + default_conf["runmode"] = RunMode.DRY_RUN + tick_interval = default_conf["ticker_interval"] + exchange = get_patched_exchange(mocker, default_conf) + exchange._klines[("XRP/BTC", tick_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.DRY_RUN + assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", tick_interval)) + assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame) + assert dp.ohlcv("UNITTEST/BTC", tick_interval) is not ticker_history + assert dp.ohlcv("UNITTEST/BTC", tick_interval, copy=False) is ticker_history + assert not dp.ohlcv("UNITTEST/BTC", tick_interval).empty + assert dp.ohlcv("NONESENSE/AAA", tick_interval).empty + + # Test with and without parameter + assert dp.ohlcv("UNITTEST/BTC", tick_interval).equals(dp.ohlcv("UNITTEST/BTC")) + + default_conf["runmode"] = RunMode.LIVE + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.LIVE + assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame) + + default_conf["runmode"] = RunMode.BACKTEST + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.BACKTEST + assert dp.ohlcv("UNITTEST/BTC", tick_interval).empty + + +def test_historic_ohlcv(mocker, default_conf, ticker_history): + + historymock = MagicMock(return_value=ticker_history) + mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) + + # exchange = get_patched_exchange(mocker, default_conf) + dp = DataProvider(default_conf, None) + data = dp.historic_ohlcv("UNITTEST/BTC", "5m") + assert isinstance(data, DataFrame) + assert historymock.call_count == 1 + assert historymock.call_args_list[0][1]["datadir"] is None + assert historymock.call_args_list[0][1]["refresh_pairs"] is False + assert historymock.call_args_list[0][1]["ticker_interval"] == "5m" + + +def test_available_pairs(mocker, default_conf, ticker_history): + exchange = get_patched_exchange(mocker, default_conf) + + tick_interval = default_conf["ticker_interval"] + exchange._klines[("XRP/BTC", tick_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history + dp = DataProvider(default_conf, exchange) + + assert len(dp.available_pairs) == 2 + assert dp.available_pairs == [ + ("XRP/BTC", tick_interval), + ("UNITTEST/BTC", tick_interval), + ] + + +def test_refresh(mocker, default_conf, ticker_history): + refresh_mock = MagicMock() + mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) + + exchange = get_patched_exchange(mocker, default_conf, id="binance") + tick_interval = default_conf["ticker_interval"] + pairs = [("XRP/BTC", tick_interval), ("UNITTEST/BTC", tick_interval)] + + pairs_non_trad = [("ETH/USDT", tick_interval), ("BTC/TUSD", "1h")] + + dp = DataProvider(default_conf, exchange) + dp.refresh(pairs) + + assert refresh_mock.call_count == 1 + assert len(refresh_mock.call_args[0]) == 1 + assert len(refresh_mock.call_args[0][0]) == len(pairs) + assert refresh_mock.call_args[0][0] == pairs + + refresh_mock.reset_mock() + dp.refresh(pairs, pairs_non_trad) + assert refresh_mock.call_count == 1 + assert len(refresh_mock.call_args[0]) == 1 + assert len(refresh_mock.call_args[0][0]) == len(pairs) + len(pairs_non_trad) + assert refresh_mock.call_args[0][0] == pairs + pairs_non_trad diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 26808e78a..b384035b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -765,7 +765,7 @@ def test_get_history(default_conf, mocker, caplog): pair = 'ETH/BTC' async def mock_candle_hist(pair, tick_interval, since_ms): - return pair, tick + return pair, tick_interval, tick exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls @@ -778,7 +778,7 @@ def test_get_history(default_conf, mocker, caplog): assert len(ret) == 2 -def test_refresh_tickers(mocker, default_conf, caplog) -> None: +def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: tick = [ [ (arrow.utcnow().timestamp - 1) * 1000, # unix timestamp ms @@ -802,12 +802,12 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: exchange = get_patched_exchange(mocker, default_conf) exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - pairs = ['IOTA/ETH', 'XRP/ETH'] + pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')] # empty dicts assert not exchange._klines - exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + exchange.refresh_latest_ohlcv(pairs) - assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) + assert log_has(f'Refreshing ohlcv data for {len(pairs)} pairs', caplog.record_tuples) assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 for pair in pairs: @@ -822,10 +822,11 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False) # test caching - exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert log_has(f"Using cached klines data for {pairs[0]} ...", caplog.record_tuples) + assert log_has(f"Using cached ohlcv data for {pairs[0][0]}, {pairs[0][1]} ...", + caplog.record_tuples) @pytest.mark.asyncio @@ -850,11 +851,12 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m") assert type(res) is tuple - assert len(res) == 2 + assert len(res) == 3 assert res[0] == pair - assert res[1] == tick + assert res[1] == "5m" + assert res[2] == tick assert exchange._api_async.fetch_ohlcv.call_count == 1 - assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + assert not log_has(f"Using cached ohlcv data for {pair} ...", caplog.record_tuples) # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), @@ -883,48 +885,14 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m") assert type(res) is tuple - assert len(res) == 2 + assert len(res) == 3 assert res[0] == pair - assert res[1] == tick + assert res[1] == "5m" + assert res[2] == tick assert exchange._api_async.fetch_ohlcv.call_count == 1 -@pytest.mark.asyncio -async def test_async_get_candles_history(default_conf, mocker): - tick = [ - [ - 1511686200000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - - async def mock_get_candle_hist(pair, tick_interval, since_ms=None): - return (pair, tick) - - exchange = get_patched_exchange(mocker, default_conf) - # Monkey-patch async function - exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - - exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist) - - pairs = ['ETH/BTC', 'XRP/BTC'] - res = await exchange.async_get_candles_history(pairs, "5m") - assert type(res) is list - assert len(res) == 2 - assert type(res[0]) is tuple - assert res[0][0] == pairs[0] - assert res[0][1] == tick - assert res[1][0] == pairs[1] - assert res[1][1] == tick - assert exchange._async_get_candle_history.call_count == 2 - - -@pytest.mark.asyncio -async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog): +def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): async def mock_get_candle_hist(pair, *args, **kwargs): if pair == 'ETH/BTC': @@ -937,12 +905,16 @@ async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog # Monkey-patch async function with empty result exchange._api_async.fetch_ohlcv = MagicMock(side_effect=mock_get_candle_hist) - pairs = ['ETH/BTC', 'XRP/BTC'] - res = await exchange.async_get_candles_history(pairs, "5m") + pairs = [("ETH/BTC", "5m"), ("XRP/BTC", "5m")] + res = exchange.refresh_latest_ohlcv(pairs) + assert exchange._klines + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert type(res) is list assert len(res) == 2 - assert type(res[0]) is tuple - assert type(res[1]) is TypeError + # Test that each is in list at least once as order is not guaranteed + assert type(res[0]) is tuple or type(res[1]) is tuple + assert type(res[0]) is TypeError or type(res[1]) is TypeError assert log_has("Error loading ETH/BTC. Result was [[]].", caplog.record_tuples) assert log_has("Async code raised an exception: TypeError", caplog.record_tuples) @@ -1010,7 +982,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker): # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' - ticks = res[1] + ticks = res[2] assert sort_mock.call_count == 1 assert ticks[0][0] == 1527830400000 @@ -1047,7 +1019,8 @@ async def test___async_get_candle_history_sort(default_conf, mocker): # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' - ticks = res[1] + assert res[1] == default_conf['ticker_interval'] + ticks = res[2] # Sorted not called again - data is already in order assert sort_mock.call_count == 0 assert ticks[0][0] == 1527827700000 diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 35ed9c49e..11d011ccd 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -18,6 +18,7 @@ from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) +from freqtrade.state import RunMode from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import SellType from freqtrade.tests.conftest import log_has, patch_exchange @@ -200,6 +201,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'timerange' not in config assert 'export' not in config + assert 'runmode' in config + assert config['runmode'] == RunMode.BACKTEST def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None: @@ -230,6 +233,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert 'exchange' in config assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config + assert config['runmode'] == RunMode.BACKTEST + assert log_has( 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples @@ -445,7 +450,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.load_data', mocked_load_data) mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) - mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -480,7 +485,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) - mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index 8ffab7f11..a58620139 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -7,6 +7,7 @@ from typing import List from freqtrade.edge import PairInfo from freqtrade.arguments import Arguments from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start) +from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has, patch_exchange @@ -26,6 +27,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> ] config = setup_configuration(get_args(args)) + assert config['runmode'] == RunMode.EDGECLI + assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -70,6 +73,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N assert 'exchange' in config assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config + assert config['runmode'] == RunMode.EDGECLI assert log_has( 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index f5c887089..67445238b 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -13,6 +13,7 @@ from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL +from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has @@ -77,6 +78,8 @@ def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> assert validated_conf['max_open_trades'] > 999999999 assert validated_conf['max_open_trades'] == float('inf') assert log_has('Validating configuration ...', caplog.record_tuples) + assert "runmode" in validated_conf + assert validated_conf['runmode'] == RunMode.DRY_RUN def test_load_config_file_exception(mocker) -> None: @@ -177,6 +180,8 @@ def test_load_config_with_params(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() assert validated_conf.get('db_url') == DEFAULT_DB_PROD_URL + assert "runmode" in validated_conf + assert validated_conf['runmode'] == RunMode.LIVE # Test args provided db_url dry_run conf = default_conf.copy() @@ -365,8 +370,9 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non args = Arguments(arglist, '').get_parsed_arg() - configuration = Configuration(args) + configuration = Configuration(args, RunMode.BACKTEST) config = configuration.get_config() + assert config['runmode'] == RunMode.BACKTEST assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -411,7 +417,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: ] args = Arguments(arglist, '').get_parsed_arg() - configuration = Configuration(args) + configuration = Configuration(args, RunMode.HYPEROPT) config = configuration.get_config() assert 'epochs' in config @@ -422,6 +428,8 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert 'spaces' in config assert config['spaces'] == ['all'] assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) + assert "runmode" in config + assert config['runmode'] == RunMode.HYPEROPT def test_check_exchange(default_conf, caplog) -> None: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1149a69e9..9200c5fa6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.refresh_tickers = lambda p, i: None + freqtrade.exchange.refresh_latest_ohlcv = lambda p: None def patch_RPCManager(mocker) -> MagicMock: @@ -807,6 +807,37 @@ def test_process_trade_no_whitelist_pair( assert result is True +def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + + def _refresh_whitelist(list): + return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] + + refresh_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + get_markets=markets, + buy=MagicMock(side_effect=TemporaryError), + refresh_latest_ohlcv=refresh_mock, + ) + inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m'), ("ETH/USDT", "1h")]) + mocker.patch('time.sleep', return_value=None) + + freqtrade = FreqtradeBot(default_conf) + freqtrade.pairlists._validate_whitelist = _refresh_whitelist + freqtrade.strategy.informative_pairs = inf_pairs + # patch_get_signal(freqtrade) + + freqtrade._process() + assert inf_pairs.call_count == 1 + assert refresh_mock.call_count == 1 + assert ("BTC/ETH", "1m") in refresh_mock.call_args[0][0] + assert ("ETH/USDT", "1h") in refresh_mock.call_args[0][0] + assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0] + + def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 4470213ef..6b954ac4c 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -161,9 +161,9 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args): tickers = {} if args.live: logger.info('Downloading pairs.') - exchange.refresh_tickers(pairs, tick_interval) + exchange.refresh_latest_ohlcv([(pair, tick_interval) for pair in pairs]) for pair in pairs: - tickers[pair] = exchange.klines(pair) + tickers[pair] = exchange.klines((pair, tick_interval)) else: tickers = history.load_data( datadir=Path(_CONF.get("datadir")), diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index c12bd966d..e2f85932f 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -29,6 +29,7 @@ from freqtrade.configuration import Configuration from freqtrade import constants from freqtrade.data import history from freqtrade.resolvers import StrategyResolver +from freqtrade.state import RunMode import freqtrade.misc as misc @@ -82,7 +83,7 @@ def plot_profit(args: Namespace) -> None: # to match the tickerdata against the profits-results timerange = Arguments.parse_timerange(args.timerange) - config = Configuration(args).get_config() + config = Configuration(args, RunMode.OTHER).get_config() # Init strategy try: diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 048b398c7..e1f7d9c11 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -67,6 +67,19 @@ class TestStrategy(IStrategy): 'sell': 'gtc' } + def informative_pairs(self): + """ + Define additional, informative pair/interval combinations to be cached from the exchange. + These pair/interval combinations are non-tradeable, unless they are part + of the whitelist as well. + For more information, please consult the documentation + :return: List of tuples in the format (pair, interval) + Sample: return [("ETH/USDT", "5m"), + ("BTC/USDT", "15m"), + ] + """ + return [] + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame