Merge pull request #1892 from freqtrade/ref/live_data

refactor `--live` handling

freqtrade/data/history.py

@@ -122,21 +122,34 @@ def load_data(datadir: Optional[Path],
               refresh_pairs: bool = False,
               exchange: Optional[Exchange] = None,
               timerange: TimeRange = TimeRange(None, None, 0, 0),
-              fill_up_missing: bool = True) -> Dict[str, DataFrame]:
+              fill_up_missing: bool = True,
+              live: bool = False
+              ) -> Dict[str, DataFrame]:
     """
     Loads ticker history data for a list of pairs the given parameters
     :return: dict(<pair>:<tickerlist>)
     """
-    result = {}
+    result: Dict[str, DataFrame] = {}
+    if live:
+        if exchange:
+            logger.info('Live: Downloading data for all defined pairs ...')
+            exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs])
+            result = {key[0]: value for key, value in exchange._klines.items() if value is not None}
+        else:
+            raise OperationalException(
+                "Exchange needs to be initialized when using live data."
+            )
+    else:
+        logger.info('Using local backtesting data ...')
 
-    for pair in pairs:
+        for pair in pairs:
-        hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,
+            hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,
-                                 datadir=datadir, timerange=timerange,
+                                     datadir=datadir, timerange=timerange,
-                                 refresh_pairs=refresh_pairs,
+                                     refresh_pairs=refresh_pairs,
-                                 exchange=exchange,
+                                     exchange=exchange,
-                                 fill_up_missing=fill_up_missing)
+                                     fill_up_missing=fill_up_missing)
-        if hist is not None:
+            if hist is not None:
-            result[pair] = hist
+                result[pair] = hist
     return result
 
 

freqtrade/optimize/backtesting.py

@@ -401,24 +401,17 @@ class Backtesting(object):
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
 
-        if self.config.get('live'):
+        timerange = Arguments.parse_timerange(None if self.config.get(
-            logger.info('Downloading data for all pairs in whitelist ...')
+            'timerange') is None else str(self.config.get('timerange')))
-            self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs])
+        data = history.load_data(
-            data = {key[0]: value for key, value in self.exchange._klines.items()}
+            datadir=Path(self.config['datadir']) if self.config.get('datadir') else None,
-
+            pairs=pairs,
-        else:
+            ticker_interval=self.ticker_interval,
-            logger.info('Using local backtesting data (using whitelist in given config) ...')
+            refresh_pairs=self.config.get('refresh_pairs', False),
-
+            exchange=self.exchange,
-            timerange = Arguments.parse_timerange(None if self.config.get(
+            timerange=timerange,
-                'timerange') is None else str(self.config.get('timerange')))
+            live=self.config.get('live', False)
-            data = history.load_data(
+        )
-                datadir=Path(self.config['datadir']) if self.config.get('datadir') else None,
-                pairs=pairs,
-                ticker_interval=self.ticker_interval,
-                refresh_pairs=self.config.get('refresh_pairs', False),
-                exchange=self.exchange,
-                timerange=timerange
-            )
 
         if not data:
             logger.critical("No data found. Terminating.")

freqtrade/tests/data/test_history.py

@@ -5,6 +5,7 @@ import os
 import uuid
 from pathlib import Path
 from shutil import copyfile
+from unittest.mock import MagicMock
 
 import arrow
 import pytest
@@ -20,7 +21,8 @@ from freqtrade.data.history import (download_pair_history,
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.strategy.default_strategy import DefaultStrategy
-from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange
+from freqtrade.tests.conftest import (get_patched_exchange, log_has,
+                                      patch_exchange)
 
 # Change this if modifying UNITTEST/BTC testdatafile
 _BTC_UNITTEST_LENGTH = 13681
@@ -136,6 +138,31 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
     _clean_test_file(file)
 
 
+def test_load_data_live(default_conf, mocker, caplog) -> None:
+    refresh_mock = MagicMock()
+    mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    history.load_data(datadir=None, ticker_interval='5m',
+                      pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
+                      live=True,
+                      exchange=exchange)
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args_list[0][0][0]) == 2
+    assert log_has('Live: Downloading data for all defined pairs ...', caplog.record_tuples)
+
+
+def test_load_data_live_noexchange(default_conf, mocker, caplog) -> None:
+
+    with pytest.raises(OperationalException,
+                       match=r'Exchange needs to be initialized when using live data.'):
+        history.load_data(datadir=None, ticker_interval='5m',
+                          pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
+                          exchange=None,
+                          live=True,
+                          )
+
+
 def test_testdata_path() -> None:
     assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None))
 

freqtrade/tests/optimize/test_backtesting.py

@@ -105,7 +105,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None:
 
 
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
-                     timerange=None, exchange=None):
+                     timerange=None, exchange=None, live=False):
     tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)}
     return pairdata
@@ -492,7 +492,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
     backtesting.start()
     # check the logs, that will contain the backtest result
     exists = [
-        'Using local backtesting data (using whitelist in given config) ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
         'Backtesting with data from 2017-11-14T21:17:00+00:00 '
@@ -857,7 +856,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
         'Using data folder: freqtrade/tests/testdata ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Downloading data for all pairs in whitelist ...',
+        'Live: Downloading data for all defined pairs ...',
         'Backtesting with data from 2017-11-14T19:31:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...'
@@ -916,7 +915,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog):
         'Using data folder: freqtrade/tests/testdata ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Downloading data for all pairs in whitelist ...',
+        'Live: Downloading data for all defined pairs ...',
         'Backtesting with data from 2017-11-14T19:31:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...',

scripts/plot_dataframe.py

@@ -139,21 +139,15 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args):
     ticker_interval = strategy.ticker_interval
     timerange = Arguments.parse_timerange(args.timerange)
 
-    tickers = {}
+    tickers = history.load_data(
-    if args.live:
+        datadir=Path(str(_CONF.get("datadir"))),
-        logger.info('Downloading pairs.')
+        pairs=pairs,
-        exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs])
+        ticker_interval=ticker_interval,
-        for pair in pairs:
+        refresh_pairs=_CONF.get('refresh_pairs', False),
-            tickers[pair] = exchange.klines((pair, ticker_interval))
+        timerange=timerange,
-    else:
+        exchange=Exchange(_CONF),
-        tickers = history.load_data(
+        live=args.live,
-            datadir=Path(str(_CONF.get("datadir"))),
+    )
-            pairs=pairs,
-            ticker_interval=ticker_interval,
-            refresh_pairs=_CONF.get('refresh_pairs', False),
-            timerange=timerange,
-            exchange=Exchange(_CONF)
-        )
 
     # No ticker found, impossible to download, len mismatch
     for pair, data in tickers.copy().items():