Merge pull request #1507 from xmatthias/feat/dataprovider

Data Provider

docs/bot-optimization.md

@@ -47,6 +47,12 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy
 **For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py)
 file as reference.**
 
+!!! Note: Strategies and Backtesting
+    To avoid problems and unexpected differences between Backtesting and dry/live modes, please be aware
+    that during backtesting the full time-interval is passed to the `populate_*()` methods at once.
+    It is therefore best to use vectorized operations (across the whole dataframe, not loops) and
+    avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle.
+
 ### Customize Indicators
 
 Buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file.
@@ -250,6 +256,95 @@ class Awesomestrategy(IStrategy):
 !!! Note:
   If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
+### Additional data (DataProvider)
+
+The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
+
+!!!Note:
+    The DataProvier is currently not available during backtesting / hyperopt, but this is planned for the future.
+
+All methods return `None` in case of failure (do not raise an exception).
+
+Please always check if the `DataProvider` is available to avoid failures during backtesting.
+
+#### Possible options for DataProvider
+
+- `available_pairs` - Property with tuples listing cached pairs with their intervals. (pair, interval)
+- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame
+- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk
+- `runmode` - Property containing the current runmode.
+
+#### ohlcv / historic_ohlcv
+
+``` python
+if self.dp:
+    if dp.runmode == 'live':
+        if ('ETH/BTC', ticker_interval) in self.dp.available_pairs:
+            data_eth = self.dp.ohlcv(pair='ETH/BTC',
+                                     ticker_interval=ticker_interval)
+    else:
+        # Get historic ohlcv data (cached on disk).
+        history_eth = self.dp.historic_ohlcv(pair='ETH/BTC',
+                                             ticker_interval='1h')
+```
+
+!!! Warning: Warning about backtesting
+    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` provides the full time-range in one go,
+    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
+
+#### Available Pairs
+
+``` python
+if self.dp:
+    for pair, ticker in self.dp.available_pairs:
+        print(f"available {pair}, {ticker}")
+```
+
+#### Get data for non-tradeable pairs
+
+Data for additional, informative pairs (reference pairs) can be beneficial for some strategies.
+Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above).
+These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting.
+
+The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument.
+
+Sample:
+
+``` python
+def informative_pairs(self):
+    return [("ETH/USDT", "5m"),
+            ("BTC/TUSD", "15m"),
+            ]
+```
+
+!!! Warning:
+    As these pairs will be refreshed as part of the regular whitelist refresh, it's best to keep this list short.
+    All intervals and all pairs can be specified as long as they are available (and active) on the used exchange.
+    It is however better to use resampling to longer time-intervals when possible
+    to avoid hammering the exchange with too many requests and risk beeing blocked.
+
+### Additional data - Wallets
+
+The strategy provides access to the `Wallets` object. This contains the current balances on the exchange.
+
+!!!NOTE:
+    Wallets is not available during backtesting / hyperopt.
+
+Please always check if `Wallets` is available to avoid failures during backtesting.
+
+``` python
+if self.wallets:
+    free_eth = self.wallets.get_free('ETH')
+    used_eth = self.wallets.get_used('ETH')
+    total_eth = self.wallets.get_total('ETH')
+```
+
+#### Possible options for Wallets
+
+- `get_free(asset)` - currently available balance to trade
+- `get_used(asset)` - currently tied up balance (open orders)
+- `get_total(asset)` - total available balance - sum of the 2 above
+
 ### Where is the default strategy?
 
 The default buy strategy is located in the file

freqtrade/configuration.py

@@ -12,6 +12,7 @@ from jsonschema import Draft4Validator, validate
 from jsonschema.exceptions import ValidationError, best_match
 
 from freqtrade import OperationalException, constants
+from freqtrade.state import RunMode
 logger = logging.getLogger(__name__)
 
 
@@ -34,9 +35,10 @@ class Configuration(object):
     Reuse this class for the bot, backtesting, hyperopt and every script that required configuration
     """
 
-    def __init__(self, args: Namespace) -> None:
+    def __init__(self, args: Namespace, runmode: RunMode = None) -> None:
         self.args = args
         self.config: Optional[Dict[str, Any]] = None
+        self.runmode = runmode
 
     def load_config(self) -> Dict[str, Any]:
         """
@@ -68,6 +70,13 @@ class Configuration(object):
         # Load Hyperopt
         config = self._load_hyperopt_config(config)
 
+        # Set runmode
+        if not self.runmode:
+            # Handle real mode, infer dry/live from config
+            self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE
+
+        config.update({'runmode': self.runmode})
+
         return config
 
     def _load_config_file(self, path: str) -> Dict[str, Any]:

freqtrade/data/dataprovider.py

@@ -0,0 +1,97 @@
+"""
+Dataprovider
+Responsible to provide data to the bot
+including Klines, tickers, historic data
+Common Interface for bot and strategy to access data.
+"""
+import logging
+from pathlib import Path
+from typing import List, Tuple
+
+from pandas import DataFrame
+
+from freqtrade.data.history import load_pair_history
+from freqtrade.exchange import Exchange
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+class DataProvider(object):
+
+    def __init__(self, config: dict, exchange: Exchange) -> None:
+        self._config = config
+        self._exchange = exchange
+
+    def refresh(self,
+                pairlist: List[Tuple[str, str]],
+                helping_pairs: List[Tuple[str, str]] = None) -> None:
+        """
+        Refresh data, called with each cycle
+        """
+        if helping_pairs:
+            self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
+        else:
+            self._exchange.refresh_latest_ohlcv(pairlist)
+
+    @property
+    def available_pairs(self) -> List[Tuple[str, str]]:
+        """
+        Return a list of tuples containing pair, tick_interval for which data is currently cached.
+        Should be whitelist + open trades.
+        """
+        return list(self._exchange._klines.keys())
+
+    def ohlcv(self, pair: str, tick_interval: str = None, copy: bool = True) -> DataFrame:
+        """
+        get ohlcv data for the given pair as DataFrame
+        Please check `available_pairs` to verify which pairs are currently cached.
+        :param pair: pair to get the data for
+        :param tick_interval: ticker_interval to get pair for
+        :param copy: copy dataframe before returning.
+                     Use false only for RO operations (where the dataframe is not modified)
+        """
+        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+            if tick_interval:
+                pairtick = (pair, tick_interval)
+            else:
+                pairtick = (pair, self._config['ticker_interval'])
+
+            return self._exchange.klines(pairtick, copy=copy)
+        else:
+            return DataFrame()
+
+    def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame:
+        """
+        get stored historic ohlcv data
+        :param pair: pair to get the data for
+        :param tick_interval: ticker_interval to get pair for
+        """
+        return load_pair_history(pair=pair,
+                                 ticker_interval=ticker_interval,
+                                 refresh_pairs=False,
+                                 datadir=Path(self._config['datadir']) if self._config.get(
+                                     'datadir') else None
+                                 )
+
+    def ticker(self, pair: str):
+        """
+        Return last ticker data
+        """
+        # TODO: Implement me
+        pass
+
+    def orderbook(self, pair: str, max: int):
+        """
+        return latest orderbook data
+        """
+        # TODO: Implement me
+        pass
+
+    @property
+    def runmode(self) -> RunMode:
+        """
+        Get runmode of the bot
+        can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other".
+        """
+        return RunMode(self._config.get('runmode', RunMode.OTHER))

freqtrade/exchange/__init__.py

@@ -80,10 +80,10 @@ class Exchange(object):
         self._cached_ticker: Dict[str, Any] = {}
 
         # Holds last candle refreshed time of each pair
-        self._pairs_last_refresh_time: Dict[str, int] = {}
+        self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {}
 
         # Holds candles
-        self._klines: Dict[str, DataFrame] = {}
+        self._klines: Dict[Tuple[str, str], DataFrame] = {}
 
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
@@ -158,11 +158,12 @@ class Exchange(object):
         """exchange ccxt id"""
         return self._api.id
 
-    def klines(self, pair: str, copy=True) -> DataFrame:
+    def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame:
-        if pair in self._klines:
+        # create key tuple
-            return self._klines[pair].copy() if copy else self._klines[pair]
+        if pair_interval in self._klines:
+            return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
         else:
-            return None
+            return DataFrame()
 
     def set_sandbox(self, api, exchange_config: dict, name: str):
         if exchange_config.get('sandbox'):
@@ -518,42 +519,41 @@ class Exchange(object):
         input_coroutines = [self._async_get_candle_history(
             pair, tick_interval, since) for since in
             range(since_ms, arrow.utcnow().timestamp * 1000, one_call)]
+
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
 
         # Combine tickers
         data: List = []
-        for p, ticker in tickers:
+        for p, ticker_interval, ticker in tickers:
             if p == pair:
                 data.extend(ticker)
-        # Sort data again after extending the result - above calls return in "async order" order
+        # Sort data again after extending the result - above calls return in "async order"
         data = sorted(data, key=lambda x: x[0])
         logger.info("downloaded %s with length %s.", pair, len(data))
         return data
 
-    def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None:
+    def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]:
         """
-        Refresh tickers asyncronously and set `_klines` of this object with the result
+        Refresh in-memory ohlcv asyncronously and set `_klines` with the result
         """
-        logger.debug("Refreshing klines for %d pairs", len(pair_list))
+        logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list))
-        asyncio.get_event_loop().run_until_complete(
-            self.async_get_candles_history(pair_list, ticker_interval))
 
-    async def async_get_candles_history(self, pairs: List[str],
-                                        tick_interval: str) -> List[Tuple[str, List]]:
-        """Download ohlcv history for pair-list asyncronously """
-        # Calculating ticker interval in second
-        interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
         input_coroutines = []
 
         # Gather corotines to run
-        for pair in pairs:
+        for pair, ticker_interval in set(pair_list):
-            if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
+            # Calculating ticker interval in second
-                    arrow.utcnow().timestamp and pair in self._klines):
+            interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60
-                input_coroutines.append(self._async_get_candle_history(pair, tick_interval))
-            else:
-                logger.debug("Using cached klines data for %s ...", pair)
 
-        tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
+            if not ((self._pairs_last_refresh_time.get((pair, ticker_interval), 0)
+                    + interval_in_sec) >= arrow.utcnow().timestamp
+                    and (pair, ticker_interval) in self._klines):
+                input_coroutines.append(self._async_get_candle_history(pair, ticker_interval))
+            else:
+                logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval)
+
+        tickers = asyncio.get_event_loop().run_until_complete(
+            asyncio.gather(*input_coroutines, return_exceptions=True))
 
         # handle caching
         for res in tickers:
@@ -561,20 +561,26 @@ class Exchange(object):
                 logger.warning("Async code raised an exception: %s", res.__class__.__name__)
                 continue
             pair = res[0]
-            ticks = res[1]
+            tick_interval = res[1]
+            ticks = res[2]
             # keeping last candle time as last refreshed time of the pair
             if ticks:
-                self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000
+                self._pairs_last_refresh_time[(pair, tick_interval)] = ticks[-1][0] // 1000
-                # keeping parsed dataframe in cache
+            # keeping parsed dataframe in cache
-                self._klines[pair] = parse_ticker_dataframe(ticks, tick_interval, fill_missing=True)
+            self._klines[(pair, tick_interval)] = parse_ticker_dataframe(
+                ticks, tick_interval, fill_missing=True)
         return tickers
 
     @retrier_async
     async def _async_get_candle_history(self, pair: str, tick_interval: str,
-                                        since_ms: Optional[int] = None) -> Tuple[str, List]:
+                                        since_ms: Optional[int] = None) -> Tuple[str, str, List]:
+        """
+        Asyncronously gets candle histories using fetch_ohlcv
+        returns tuple: (pair, tick_interval, ohlcv_list)
+        """
         try:
             # fetch ohlcv asynchronously
-            logger.debug("fetching %s since %s ...", pair, since_ms)
+            logger.debug("fetching %s, %s since %s ...", pair, tick_interval, since_ms)
 
             data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval,
                                                      since=since_ms)
@@ -588,9 +594,9 @@ class Exchange(object):
                     data = sorted(data, key=lambda x: x[0])
             except IndexError:
                 logger.exception("Error loading %s. Result was %s.", pair, data)
-                return pair, []
+                return pair, tick_interval, []
-            logger.debug("done fetching %s ...", pair)
+            logger.debug("done fetching %s, %s ...", pair, tick_interval)
-            return pair, data
+            return pair, tick_interval, data
 
         except ccxt.NotSupported as e:
             raise OperationalException(

freqtrade/freqtradebot.py

@@ -15,6 +15,7 @@ from requests.exceptions import RequestException
 from freqtrade import (DependencyException, OperationalException,
                        TemporaryError, __version__, constants, persistence)
 from freqtrade.data.converter import order_book_to_dataframe
+from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
 from freqtrade.exchange import Exchange
 from freqtrade.persistence import Trade
@@ -36,9 +37,9 @@ class FreqtradeBot(object):
 
     def __init__(self, config: Dict[str, Any]) -> None:
         """
-        Init all variables and object the bot need to work
+        Init all variables and objects the bot needs to work
-        :param config: configuration dict, you can use the Configuration.get_config()
+        :param config: configuration dict, you can use Configuration.get_config()
-        method to get the config dict.
+        to get the config dict.
         """
 
         logger.info(
@@ -54,9 +55,15 @@ class FreqtradeBot(object):
         self.strategy: IStrategy = StrategyResolver(self.config).strategy
 
         self.rpc: RPCManager = RPCManager(self)
-        self.persistence = None
         self.exchange = Exchange(self.config)
         self.wallets = Wallets(self.exchange)
+        self.dataprovider = DataProvider(self.config, self.exchange)
+
+        # Attach Dataprovider to Strategy baseclass
+        IStrategy.dp = self.dataprovider
+        # Attach Wallets to Strategy baseclass
+        IStrategy.wallets = self.wallets
+
         pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList')
         self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist
 
@@ -151,9 +158,6 @@ class FreqtradeBot(object):
             self.active_pair_whitelist = self.pairlists.whitelist
 
             # Calculating Edge positiong
-            # Should be called before refresh_tickers
-            # Otherwise it will override cached klines in exchange
-            # with delta value (klines only from last refresh_pairs)
             if self.edge:
                 self.edge.calculate()
                 self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist)
@@ -166,8 +170,12 @@ class FreqtradeBot(object):
             self.active_pair_whitelist.extend([trade.pair for trade in trades
                                                if trade.pair not in self.active_pair_whitelist])
 
+            # Create pair-whitelist tuple with (pair, ticker_interval)
+            pair_whitelist_tuple = [(pair, self.config['ticker_interval'])
+                                    for pair in self.active_pair_whitelist]
             # Refreshing candles
-            self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval)
+            self.dataprovider.refresh(pair_whitelist_tuple,
+                                      self.strategy.informative_pairs())
 
             # First process current opened trades
             for trade in trades:
@@ -317,7 +325,9 @@ class FreqtradeBot(object):
 
         # running get_signal on historical data fetched
         for _pair in whitelist:
-            (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines(_pair))
+            (buy, sell) = self.strategy.get_signal(
+                _pair, interval, self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval))
+
             if buy and not sell:
                 stake_amount = self._get_trade_stake_amount(_pair)
                 if not stake_amount:
@@ -578,8 +588,9 @@ class FreqtradeBot(object):
         (buy, sell) = (False, False)
         experimental = self.config.get('experimental', {})
         if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
-            (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
+            (buy, sell) = self.strategy.get_signal(
-                                                   self.exchange.klines(trade.pair))
+                trade.pair, self.strategy.ticker_interval,
+                self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval))
 
         config_ask_strategy = self.config.get('ask_strategy', {})
         if config_ask_strategy.get('use_order_book', False):

freqtrade/main.py

@@ -39,7 +39,7 @@ def main(sysargv: List[str]) -> None:
     return_code = 1
     try:
         # Load and validate configuration
-        config = Configuration(args).get_config()
+        config = Configuration(args, None).get_config()
 
         # Init the bot
         freqtrade = FreqtradeBot(config)
@@ -76,7 +76,7 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot:
     freqtrade.cleanup()
 
     # Create new instance
-    freqtrade = FreqtradeBot(Configuration(args).get_config())
+    freqtrade = FreqtradeBot(Configuration(args, None).get_config())
     freqtrade.rpc.send_msg({
         'type': RPCMessageType.STATUS_NOTIFICATION,
         'status': 'config reloaded'

freqtrade/optimize/backtesting.py

@@ -22,6 +22,7 @@ from freqtrade.data import history
 from freqtrade.misc import file_dump_json
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import StrategyResolver
+from freqtrade.state import RunMode
 from freqtrade.strategy.interface import SellType, IStrategy
 
 logger = logging.getLogger(__name__)
@@ -374,8 +375,9 @@ class Backtesting(object):
 
         if self.config.get('live'):
             logger.info('Downloading data for all pairs in whitelist ...')
-            self.exchange.refresh_tickers(pairs, self.ticker_interval)
+            self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs])
-            data = self.exchange._klines
+            data = {key[0]: value for key, value in self.exchange._klines.items()}
+
         else:
             logger.info('Using local backtesting data (using whitelist in given config) ...')
 
@@ -459,7 +461,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]:
     :param args: Cli args from Arguments()
     :return: Configuration
     """
-    configuration = Configuration(args)
+    configuration = Configuration(args, RunMode.BACKTEST)
     config = configuration.get_config()
 
     # Ensure we do not use Exchange credentials

freqtrade/optimize/edge_cli.py

@@ -9,10 +9,11 @@ from typing import Dict, Any
 from tabulate import tabulate
 from freqtrade.edge import Edge
 
-from freqtrade.configuration import Configuration
 from freqtrade.arguments import Arguments
+from freqtrade.configuration import Configuration
 from freqtrade.exchange import Exchange
 from freqtrade.resolvers import StrategyResolver
+from freqtrade.state import RunMode
 
 logger = logging.getLogger(__name__)
 
@@ -83,7 +84,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]:
     :param args: Cli args from Arguments()
     :return: Configuration
     """
-    configuration = Configuration(args)
+    configuration = Configuration(args, RunMode.EDGECLI)
     config = configuration.get_config()
 
     # Ensure we do not use Exchange credentials

freqtrade/optimize/hyperopt.py

@@ -25,6 +25,7 @@ from freqtrade.configuration import Configuration
 from freqtrade.data.history import load_data
 from freqtrade.optimize import get_timeframe
 from freqtrade.optimize.backtesting import Backtesting
+from freqtrade.state import RunMode
 from freqtrade.resolvers import HyperOptResolver
 
 logger = logging.getLogger(__name__)
@@ -306,7 +307,7 @@ def start(args: Namespace) -> None:
 
     # Initialize configuration
     # Monkey patch the configuration with hyperopt_conf.py
-    configuration = Configuration(args)
+    configuration = Configuration(args, RunMode.HYPEROPT)
     logger.info('Starting freqtrade in Hyperopt mode')
     config = configuration.load_config()
 

freqtrade/resolvers/strategy_resolver.py

@@ -3,11 +3,11 @@
 """
 This module load custom strategies
 """
-import inspect
 import logging
 import tempfile
 from base64 import urlsafe_b64decode
 from collections import OrderedDict
+from inspect import getfullargspec
 from pathlib import Path
 from typing import Dict, Optional
 
@@ -126,11 +126,9 @@ class StrategyResolver(IResolver):
                 if strategy:
                     logger.info('Using resolved strategy %s from \'%s\'', strategy_name, _path)
                     strategy._populate_fun_len = len(
-                        inspect.getfullargspec(strategy.populate_indicators).args)
+                        getfullargspec(strategy.populate_indicators).args)
-                    strategy._buy_fun_len = len(
+                    strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
-                        inspect.getfullargspec(strategy.populate_buy_trend).args)
+                    strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
-                    strategy._sell_fun_len = len(
-                        inspect.getfullargspec(strategy.populate_sell_trend).args)
 
                     return import_strategy(strategy, config=config)
             except FileNotFoundError:

freqtrade/state.py

@@ -3,13 +3,26 @@
 """
 Bot state constant
 """
-import enum
+from enum import Enum
 
 
-class State(enum.Enum):
+class State(Enum):
     """
     Bot application states
     """
-    RUNNING = 0
+    RUNNING = 1
-    STOPPED = 1
+    STOPPED = 2
-    RELOAD_CONF = 2
+    RELOAD_CONF = 3
+
+
+class RunMode(Enum):
+    """
+    Bot running mode (backtest, hyperopt, ...)
+    can be "live", "dry-run", "backtest", "edgecli", "hyperopt".
+    """
+    LIVE = "live"
+    DRY_RUN = "dry_run"
+    BACKTEST = "backtest"
+    EDGECLI = "edgecli"
+    HYPEROPT = "hyperopt"
+    OTHER = "other"  # Used for plotting scripts and test

freqtrade/strategy/default_strategy.py

@@ -42,6 +42,19 @@ class DefaultStrategy(IStrategy):
         'sell': 'gtc',
     }
 
+    def informative_pairs(self):
+        """
+        Define additional, informative pair/interval combinations to be cached from the exchange.
+        These pair/interval combinations are non-tradeable, unless they are part
+        of the whitelist as well.
+        For more information, please consult the documentation
+        :return: List of tuples in the format (pair, interval)
+            Sample: return [("ETH/USDT", "5m"),
+                            ("BTC/USDT", "15m"),
+                            ]
+        """
+        return []
+
     def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
         Adds several different TA indicators to the given DataFrame

freqtrade/strategy/interface.py

@@ -13,7 +13,9 @@ import arrow
 from pandas import DataFrame
 
 from freqtrade import constants
+from freqtrade.data.dataprovider import DataProvider
 from freqtrade.persistence import Trade
+from freqtrade.wallets import Wallets
 
 logger = logging.getLogger(__name__)
 
@@ -93,12 +95,16 @@ class IStrategy(ABC):
     # run "populate_indicators" only for new candle
     process_only_new_candles: bool = False
 
-    # Dict to determine if analysis is necessary
+    # Class level variables (intentional) containing
-    _last_candle_seen_per_pair: Dict[str, datetime] = {}
+    # the dataprovider (dp) (access to other candles, historic data, ...)
+    # and wallets - access to the current balance.
+    dp: DataProvider
+    wallets: Wallets
 
     def __init__(self, config: dict) -> None:
         self.config = config
-        self._last_candle_seen_per_pair = {}
+        # Dict to determine if analysis is necessary
+        self._last_candle_seen_per_pair: Dict[str, datetime] = {}
 
     @abstractmethod
     def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
@@ -127,6 +133,19 @@ class IStrategy(ABC):
         :return: DataFrame with sell column
         """
 
+    def informative_pairs(self) -> List[Tuple[str, str]]:
+        """
+        Define additional, informative pair/interval combinations to be cached from the exchange.
+        These pair/interval combinations are non-tradeable, unless they are part
+        of the whitelist as well.
+        For more information, please consult the documentation
+        :return: List of tuples in the format (pair, interval)
+            Sample: return [("ETH/USDT", "5m"),
+                            ("BTC/USDT", "15m"),
+                            ]
+        """
+        return []
+
     def get_strategy_name(self) -> str:
         """
         Returns strategy class name

freqtrade/tests/data/test_dataprovider.py

@@ -0,0 +1,92 @@
+from unittest.mock import MagicMock
+
+from pandas import DataFrame
+
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.state import RunMode
+from freqtrade.tests.conftest import get_patched_exchange
+
+
+def test_ohlcv(mocker, default_conf, ticker_history):
+    default_conf["runmode"] = RunMode.DRY_RUN
+    tick_interval = default_conf["ticker_interval"]
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._klines[("XRP/BTC", tick_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.DRY_RUN
+    assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", tick_interval))
+    assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame)
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval) is not ticker_history
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval, copy=False) is ticker_history
+    assert not dp.ohlcv("UNITTEST/BTC", tick_interval).empty
+    assert dp.ohlcv("NONESENSE/AAA", tick_interval).empty
+
+    # Test with and without parameter
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval).equals(dp.ohlcv("UNITTEST/BTC"))
+
+    default_conf["runmode"] = RunMode.LIVE
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.LIVE
+    assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame)
+
+    default_conf["runmode"] = RunMode.BACKTEST
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.BACKTEST
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval).empty
+
+
+def test_historic_ohlcv(mocker, default_conf, ticker_history):
+
+    historymock = MagicMock(return_value=ticker_history)
+    mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
+
+    # exchange = get_patched_exchange(mocker, default_conf)
+    dp = DataProvider(default_conf, None)
+    data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
+    assert isinstance(data, DataFrame)
+    assert historymock.call_count == 1
+    assert historymock.call_args_list[0][1]["datadir"] is None
+    assert historymock.call_args_list[0][1]["refresh_pairs"] is False
+    assert historymock.call_args_list[0][1]["ticker_interval"] == "5m"
+
+
+def test_available_pairs(mocker, default_conf, ticker_history):
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    tick_interval = default_conf["ticker_interval"]
+    exchange._klines[("XRP/BTC", tick_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history
+    dp = DataProvider(default_conf, exchange)
+
+    assert len(dp.available_pairs) == 2
+    assert dp.available_pairs == [
+        ("XRP/BTC", tick_interval),
+        ("UNITTEST/BTC", tick_interval),
+    ]
+
+
+def test_refresh(mocker, default_conf, ticker_history):
+    refresh_mock = MagicMock()
+    mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
+
+    exchange = get_patched_exchange(mocker, default_conf, id="binance")
+    tick_interval = default_conf["ticker_interval"]
+    pairs = [("XRP/BTC", tick_interval), ("UNITTEST/BTC", tick_interval)]
+
+    pairs_non_trad = [("ETH/USDT", tick_interval), ("BTC/TUSD", "1h")]
+
+    dp = DataProvider(default_conf, exchange)
+    dp.refresh(pairs)
+
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args[0]) == 1
+    assert len(refresh_mock.call_args[0][0]) == len(pairs)
+    assert refresh_mock.call_args[0][0] == pairs
+
+    refresh_mock.reset_mock()
+    dp.refresh(pairs, pairs_non_trad)
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args[0]) == 1
+    assert len(refresh_mock.call_args[0][0]) == len(pairs) + len(pairs_non_trad)
+    assert refresh_mock.call_args[0][0] == pairs + pairs_non_trad

freqtrade/tests/exchange/test_exchange.py

@@ -765,7 +765,7 @@ def test_get_history(default_conf, mocker, caplog):
     pair = 'ETH/BTC'
 
     async def mock_candle_hist(pair, tick_interval, since_ms):
-        return pair, tick
+        return pair, tick_interval, tick
 
     exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
     # one_call calculation * 1.8 should do 2 calls
@@ -778,7 +778,7 @@ def test_get_history(default_conf, mocker, caplog):
     assert len(ret) == 2
 
 
-def test_refresh_tickers(mocker, default_conf, caplog) -> None:
+def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
     tick = [
         [
             (arrow.utcnow().timestamp - 1) * 1000,  # unix timestamp ms
@@ -802,12 +802,12 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None:
     exchange = get_patched_exchange(mocker, default_conf)
     exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
 
-    pairs = ['IOTA/ETH', 'XRP/ETH']
+    pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]
     # empty dicts
     assert not exchange._klines
-    exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
+    exchange.refresh_latest_ohlcv(pairs)
 
-    assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
+    assert log_has(f'Refreshing ohlcv data for {len(pairs)} pairs', caplog.record_tuples)
     assert exchange._klines
     assert exchange._api_async.fetch_ohlcv.call_count == 2
     for pair in pairs:
@@ -822,10 +822,11 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None:
         assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False)
 
     # test caching
-    exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
+    exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')])
 
     assert exchange._api_async.fetch_ohlcv.call_count == 2
-    assert log_has(f"Using cached klines data for {pairs[0]} ...", caplog.record_tuples)
+    assert log_has(f"Using cached ohlcv data for {pairs[0][0]}, {pairs[0][1]} ...",
+                   caplog.record_tuples)
 
 
 @pytest.mark.asyncio
@@ -850,11 +851,12 @@ async def test__async_get_candle_history(default_conf, mocker, caplog):
     pair = 'ETH/BTC'
     res = await exchange._async_get_candle_history(pair, "5m")
     assert type(res) is tuple
-    assert len(res) == 2
+    assert len(res) == 3
     assert res[0] == pair
-    assert res[1] == tick
+    assert res[1] == "5m"
+    assert res[2] == tick
     assert exchange._api_async.fetch_ohlcv.call_count == 1
-    assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
+    assert not log_has(f"Using cached ohlcv data for {pair} ...", caplog.record_tuples)
 
     # exchange = Exchange(default_conf)
     await async_ccxt_exception(mocker, default_conf, MagicMock(),
@@ -883,48 +885,14 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
     pair = 'ETH/BTC'
     res = await exchange._async_get_candle_history(pair, "5m")
     assert type(res) is tuple
-    assert len(res) == 2
+    assert len(res) == 3
     assert res[0] == pair
-    assert res[1] == tick
+    assert res[1] == "5m"
+    assert res[2] == tick
     assert exchange._api_async.fetch_ohlcv.call_count == 1
 
 
-@pytest.mark.asyncio
+def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog):
-async def test_async_get_candles_history(default_conf, mocker):
-    tick = [
-        [
-            1511686200000,  # unix timestamp ms
-            1,  # open
-            2,  # high
-            3,  # low
-            4,  # close
-            5,  # volume (in quote currency)
-        ]
-    ]
-
-    async def mock_get_candle_hist(pair, tick_interval, since_ms=None):
-        return (pair, tick)
-
-    exchange = get_patched_exchange(mocker, default_conf)
-    # Monkey-patch async function
-    exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
-
-    exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist)
-
-    pairs = ['ETH/BTC', 'XRP/BTC']
-    res = await exchange.async_get_candles_history(pairs, "5m")
-    assert type(res) is list
-    assert len(res) == 2
-    assert type(res[0]) is tuple
-    assert res[0][0] == pairs[0]
-    assert res[0][1] == tick
-    assert res[1][0] == pairs[1]
-    assert res[1][1] == tick
-    assert exchange._async_get_candle_history.call_count == 2
-
-
-@pytest.mark.asyncio
-async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog):
 
     async def mock_get_candle_hist(pair, *args, **kwargs):
         if pair == 'ETH/BTC':
@@ -937,12 +905,16 @@ async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog
     # Monkey-patch async function with empty result
     exchange._api_async.fetch_ohlcv = MagicMock(side_effect=mock_get_candle_hist)
 
-    pairs = ['ETH/BTC', 'XRP/BTC']
+    pairs = [("ETH/BTC", "5m"), ("XRP/BTC", "5m")]
-    res = await exchange.async_get_candles_history(pairs, "5m")
+    res = exchange.refresh_latest_ohlcv(pairs)
+    assert exchange._klines
+    assert exchange._api_async.fetch_ohlcv.call_count == 2
+
     assert type(res) is list
     assert len(res) == 2
-    assert type(res[0]) is tuple
+    # Test that each is in list at least once as order is not guaranteed
-    assert type(res[1]) is TypeError
+    assert type(res[0]) is tuple or type(res[1]) is tuple
+    assert type(res[0]) is TypeError or type(res[1]) is TypeError
     assert log_has("Error loading ETH/BTC. Result was [[]].", caplog.record_tuples)
     assert log_has("Async code raised an exception: TypeError", caplog.record_tuples)
 
@@ -1010,7 +982,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker):
     # Test the ticker history sort
     res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval'])
     assert res[0] == 'ETH/BTC'
-    ticks = res[1]
+    ticks = res[2]
 
     assert sort_mock.call_count == 1
     assert ticks[0][0] == 1527830400000
@@ -1047,7 +1019,8 @@ async def test___async_get_candle_history_sort(default_conf, mocker):
     # Test the ticker history sort
     res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval'])
     assert res[0] == 'ETH/BTC'
-    ticks = res[1]
+    assert res[1] == default_conf['ticker_interval']
+    ticks = res[2]
     # Sorted not called again - data is already in order
     assert sort_mock.call_count == 0
     assert ticks[0][0] == 1527827700000

freqtrade/tests/optimize/test_backtesting.py

@@ -18,6 +18,7 @@ from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.optimize import get_timeframe
 from freqtrade.optimize.backtesting import (Backtesting, setup_configuration,
                                             start)
+from freqtrade.state import RunMode
 from freqtrade.strategy.default_strategy import DefaultStrategy
 from freqtrade.strategy.interface import SellType
 from freqtrade.tests.conftest import log_has, patch_exchange
@@ -200,6 +201,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
 
     assert 'timerange' not in config
     assert 'export' not in config
+    assert 'runmode' in config
+    assert config['runmode'] == RunMode.BACKTEST
 
 
 def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
@@ -230,6 +233,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
     assert 'exchange' in config
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
+    assert config['runmode'] == RunMode.BACKTEST
+
     assert log_has(
         'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples
@@ -445,7 +450,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
 
     mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
     mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe)
-    mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.optimize.backtesting.Backtesting',
@@ -480,7 +485,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None:
 
     mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={}))
     mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe)
-    mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.optimize.backtesting.Backtesting',

freqtrade/tests/optimize/test_edge_cli.py

@@ -7,6 +7,7 @@ from typing import List
 from freqtrade.edge import PairInfo
 from freqtrade.arguments import Arguments
 from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start)
+from freqtrade.state import RunMode
 from freqtrade.tests.conftest import log_has, patch_exchange
 
 
@@ -26,6 +27,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     ]
 
     config = setup_configuration(get_args(args))
+    assert config['runmode'] == RunMode.EDGECLI
+
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config
@@ -70,6 +73,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N
     assert 'exchange' in config
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
+    assert config['runmode'] == RunMode.EDGECLI
     assert log_has(
         'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples

freqtrade/tests/test_configuration.py

@@ -13,6 +13,7 @@ from freqtrade import OperationalException
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration, set_loggers
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
+from freqtrade.state import RunMode
 from freqtrade.tests.conftest import log_has
 
 
@@ -77,6 +78,8 @@ def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) ->
     assert validated_conf['max_open_trades'] > 999999999
     assert validated_conf['max_open_trades'] == float('inf')
     assert log_has('Validating configuration ...', caplog.record_tuples)
+    assert "runmode" in validated_conf
+    assert validated_conf['runmode'] == RunMode.DRY_RUN
 
 
 def test_load_config_file_exception(mocker) -> None:
@@ -177,6 +180,8 @@ def test_load_config_with_params(default_conf, mocker) -> None:
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
     assert validated_conf.get('db_url') == DEFAULT_DB_PROD_URL
+    assert "runmode" in validated_conf
+    assert validated_conf['runmode'] == RunMode.LIVE
 
     # Test args provided db_url dry_run
     conf = default_conf.copy()
@@ -365,8 +370,9 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non
 
     args = Arguments(arglist, '').get_parsed_arg()
 
-    configuration = Configuration(args)
+    configuration = Configuration(args, RunMode.BACKTEST)
     config = configuration.get_config()
+    assert config['runmode'] == RunMode.BACKTEST
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config
@@ -411,7 +417,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
     ]
     args = Arguments(arglist, '').get_parsed_arg()
 
-    configuration = Configuration(args)
+    configuration = Configuration(args, RunMode.HYPEROPT)
     config = configuration.get_config()
 
     assert 'epochs' in config
@@ -422,6 +428,8 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
     assert 'spaces' in config
     assert config['spaces'] == ['all']
     assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples)
+    assert "runmode" in config
+    assert config['runmode'] == RunMode.HYPEROPT
 
 
 def test_check_exchange(default_conf, caplog) -> None:

freqtrade/tests/test_freqtradebot.py

@@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     :return: None
     """
     freqtrade.strategy.get_signal = lambda e, s, t: value
-    freqtrade.exchange.refresh_tickers = lambda p, i: None
+    freqtrade.exchange.refresh_latest_ohlcv = lambda p: None
 
 
 def patch_RPCManager(mocker) -> MagicMock:
@@ -807,6 +807,37 @@ def test_process_trade_no_whitelist_pair(
     assert result is True
 
 
+def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+
+    def _refresh_whitelist(list):
+        return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC']
+
+    refresh_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_ticker=ticker,
+        get_markets=markets,
+        buy=MagicMock(side_effect=TemporaryError),
+        refresh_latest_ohlcv=refresh_mock,
+    )
+    inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m'), ("ETH/USDT", "1h")])
+    mocker.patch('time.sleep', return_value=None)
+
+    freqtrade = FreqtradeBot(default_conf)
+    freqtrade.pairlists._validate_whitelist = _refresh_whitelist
+    freqtrade.strategy.informative_pairs = inf_pairs
+    # patch_get_signal(freqtrade)
+
+    freqtrade._process()
+    assert inf_pairs.call_count == 1
+    assert refresh_mock.call_count == 1
+    assert ("BTC/ETH", "1m") in refresh_mock.call_args[0][0]
+    assert ("ETH/USDT", "1h") in refresh_mock.call_args[0][0]
+    assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0]
+
+
 def test_balance_fully_ask_side(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 0.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)

scripts/plot_dataframe.py

@@ -161,9 +161,9 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args):
     tickers = {}
     if args.live:
         logger.info('Downloading pairs.')
-        exchange.refresh_tickers(pairs, tick_interval)
+        exchange.refresh_latest_ohlcv([(pair, tick_interval) for pair in pairs])
         for pair in pairs:
-            tickers[pair] = exchange.klines(pair)
+            tickers[pair] = exchange.klines((pair, tick_interval))
     else:
         tickers = history.load_data(
             datadir=Path(_CONF.get("datadir")),

scripts/plot_profit.py

@@ -29,6 +29,7 @@ from freqtrade.configuration import Configuration
 from freqtrade import constants
 from freqtrade.data import history
 from freqtrade.resolvers import StrategyResolver
+from freqtrade.state import RunMode
 import freqtrade.misc as misc
 
 
@@ -82,7 +83,7 @@ def plot_profit(args: Namespace) -> None:
     # to match the tickerdata against the profits-results
     timerange = Arguments.parse_timerange(args.timerange)
 
-    config = Configuration(args).get_config()
+    config = Configuration(args, RunMode.OTHER).get_config()
 
     # Init strategy
     try:

user_data/strategies/test_strategy.py

@@ -67,6 +67,19 @@ class TestStrategy(IStrategy):
         'sell': 'gtc'
     }
 
+    def informative_pairs(self):
+        """
+        Define additional, informative pair/interval combinations to be cached from the exchange.
+        These pair/interval combinations are non-tradeable, unless they are part
+        of the whitelist as well.
+        For more information, please consult the documentation
+        :return: List of tuples in the format (pair, interval)
+            Sample: return [("ETH/USDT", "5m"),
+                            ("BTC/USDT", "15m"),
+                            ]
+        """
+        return []
+
     def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
         Adds several different TA indicators to the given DataFrame