Merge pull request #1404 from freqtrade/feat/pass_df

keep DF instead of list

freqtrade/exchange/__init__.py

@@ -7,12 +7,14 @@ from typing import List, Dict, Tuple, Any, Optional
 from datetime import datetime
 from math import floor, ceil
 
+import arrow
 import asyncio
 import ccxt
 import ccxt.async_support as ccxt_async
-import arrow
+from pandas import DataFrame
 
 from freqtrade import constants, OperationalException, DependencyException, TemporaryError
+from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe
 
 logger = logging.getLogger(__name__)
 
@@ -81,7 +83,7 @@ class Exchange(object):
         self._pairs_last_refresh_time: Dict[str, int] = {}
 
         # Holds candles
-        self.klines: Dict[str, Any] = {}
+        self._klines: Dict[str, DataFrame] = {}
 
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
@@ -155,6 +157,12 @@ class Exchange(object):
         """exchange ccxt id"""
         return self._api.id
 
+    def klines(self, pair: str) -> DataFrame:
+        if pair in self._klines:
+            return self._klines[pair].copy()
+        else:
+            return None
+
     def set_sandbox(self, api, exchange_config: dict, name: str):
         if exchange_config.get('sandbox'):
             if api.urls.get('test'):
@@ -499,7 +507,7 @@ class Exchange(object):
 
     def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None:
         """
-        Refresh tickers asyncronously and set `klines` of this object with the result
+        Refresh tickers asyncronously and set `_klines` of this object with the result
         """
         logger.debug("Refreshing klines for %d pairs", len(pair_list))
         asyncio.get_event_loop().run_until_complete(
@@ -515,7 +523,7 @@ class Exchange(object):
         # Gather corotines to run
         for pair in pairs:
             if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
-                    arrow.utcnow().timestamp and pair in self.klines):
+                    arrow.utcnow().timestamp and pair in self._klines):
                 input_coroutines.append(self._async_get_candle_history(pair, tick_interval))
             else:
                 logger.debug("Using cached klines data for %s ...", pair)
@@ -528,7 +536,7 @@ class Exchange(object):
             if ticks:
                 self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000
             # keeping parsed dataframe in cache
-            self.klines[pair] = ticks
+            self._klines[pair] = parse_ticker_dataframe(ticks)
         return tickers
 
     @retrier_async

freqtrade/exchange/exchange_helpers.py

@@ -14,6 +14,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame:
     :param ticker: ticker list, as returned by exchange.async_get_candle_history
     :return: DataFrame
     """
+    logger.debug("Parsing tickerlist to dataframe")
     cols = ['date', 'open', 'high', 'low', 'close', 'volume']
     frame = DataFrame(ticker, columns=cols)
 

freqtrade/freqtradebot.py

@@ -317,7 +317,7 @@ class FreqtradeBot(object):
 
         # running get_signal on historical data fetched
         for _pair in whitelist:
-            (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair))
+            (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines(_pair))
             if buy and not sell:
                 stake_amount = self._get_trade_stake_amount(_pair)
                 if not stake_amount:
@@ -540,9 +540,8 @@ class FreqtradeBot(object):
         (buy, sell) = (False, False)
         experimental = self.config.get('experimental', {})
         if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
-            ticker = self.exchange.klines.get(trade.pair)
             (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
-                                                   ticker)
+                                                   self.exchange.klines(trade.pair))
 
         config_ask_strategy = self.config.get('ask_strategy', {})
         if config_ask_strategy.get('use_order_book', False):

freqtrade/optimize/backtesting.py

@@ -362,7 +362,7 @@ class Backtesting(object):
         if self.config.get('live'):
             logger.info('Downloading data for all pairs in whitelist ...')
             self.exchange.refresh_tickers(pairs, self.ticker_interval)
-            data = self.exchange.klines
+            data = self.exchange._klines
         else:
             logger.info('Using local backtesting data (using whitelist in given config) ...')
 

freqtrade/strategy/interface.py

@@ -6,7 +6,7 @@ import logging
 from abc import ABC, abstractmethod
 from datetime import datetime
 from enum import Enum
-from typing import Dict, List, NamedTuple, Optional, Tuple
+from typing import Dict, List, NamedTuple, Tuple
 import warnings
 
 import arrow
@@ -122,19 +122,17 @@ class IStrategy(ABC):
         """
         return self.__class__.__name__
 
-    def analyze_ticker(self, ticker_history: List[Dict], metadata: dict) -> DataFrame:
+    def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
         Parses the given ticker history and returns a populated DataFrame
         add several TA indicators and buy signal to it
         :return DataFrame with ticker data and indicator data
         """
 
-        dataframe = parse_ticker_dataframe(ticker_history)
-
         pair = str(metadata.get('pair'))
 
         # Test if seen this pair and last candle before.
-        # always run if process_only_new_candles is set to true
+        # always run if process_only_new_candles is set to false
         if (not self.process_only_new_candles or
                 self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']):
             # Defs that only make change on new candle data.
@@ -155,19 +153,20 @@ class IStrategy(ABC):
         return dataframe
 
     def get_signal(self, pair: str, interval: str,
-                   ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]:
+                   dataframe: DataFrame) -> Tuple[bool, bool]:
         """
         Calculates current signal based several technical analysis indicators
         :param pair: pair in format ANT/BTC
         :param interval: Interval to use (in min)
+        :param dataframe: Dataframe to analyze
         :return: (Buy, Sell) A bool-tuple indicating buy/sell signal
         """
-        if not ticker_hist:
+        if not isinstance(dataframe, DataFrame) or dataframe.empty:
             logger.warning('Empty ticker history for pair %s', pair)
             return False, False
 
         try:
-            dataframe = self.analyze_ticker(ticker_hist, {'pair': pair})
+            dataframe = self.analyze_ticker(dataframe, {'pair': pair})
         except ValueError as error:
             logger.warning(
                 'Unable to analyze ticker for pair %s: %s',

freqtrade/tests/conftest.py

@@ -481,7 +481,7 @@ def order_book_l2():
 
 
 @pytest.fixture
-def ticker_history():
+def ticker_history_list():
     return [
         [
             1511686200000,  # unix timestamp ms
@@ -510,6 +510,11 @@ def ticker_history():
     ]
 
 
+@pytest.fixture
+def ticker_history(ticker_history_list):
+    return parse_ticker_dataframe(ticker_history_list)
+
+
 @pytest.fixture
 def tickers():
     return MagicMock(return_value={

freqtrade/tests/exchange/test_exchange.py

@@ -9,6 +9,7 @@ from unittest.mock import Mock, MagicMock, PropertyMock
 import arrow
 import ccxt
 import pytest
+from pandas import DataFrame
 
 from freqtrade import DependencyException, OperationalException, TemporaryError
 from freqtrade.exchange import API_RETRY_COUNT, Exchange
@@ -737,12 +738,20 @@ def test_get_history(default_conf, mocker, caplog):
 def test_refresh_tickers(mocker, default_conf, caplog) -> None:
     tick = [
         [
-            arrow.utcnow().timestamp * 1000,  # unix timestamp ms
+            (arrow.utcnow().timestamp - 1) * 1000,  # unix timestamp ms
             1,  # open
             2,  # high
             3,  # low
             4,  # close
             5,  # volume (in quote currency)
+        ],
+        [
+            arrow.utcnow().timestamp * 1000,  # unix timestamp ms
+            3,  # open
+            1,  # high
+            4,  # low
+            6,  # close
+            5,  # volume (in quote currency)
         ]
     ]
 
@@ -752,14 +761,15 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None:
 
     pairs = ['IOTA/ETH', 'XRP/ETH']
     # empty dicts
-    assert not exchange.klines
+    assert not exchange._klines
     exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
 
     assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
-    assert exchange.klines
+    assert exchange._klines
     assert exchange._api_async.fetch_ohlcv.call_count == 2
     for pair in pairs:
-        assert exchange.klines[pair]
+        assert isinstance(exchange.klines(pair), DataFrame)
+        assert len(exchange.klines(pair)) > 0
 
     # test caching
     exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')

freqtrade/tests/exchange/test_exchange_helpers.py

@@ -1,6 +1,8 @@
 # pragma pylint: disable=missing-docstring, C0103
+import logging
 
 from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe
+from freqtrade.tests.conftest import log_has
 
 
 def test_dataframe_correct_length(result):
@@ -13,9 +15,11 @@ def test_dataframe_correct_columns(result):
         ['date', 'open', 'high', 'low', 'close', 'volume']
 
 
-def test_parse_ticker_dataframe(ticker_history):
+def test_parse_ticker_dataframe(ticker_history, caplog):
     columns = ['date', 'open', 'high', 'low', 'close', 'volume']
 
+    caplog.set_level(logging.DEBUG)
     # Test file with BV data
     dataframe = parse_ticker_dataframe(ticker_history)
     assert dataframe.columns.tolist() == columns
+    assert log_has('Parsing tickerlist to dataframe', caplog.record_tuples)

freqtrade/tests/optimize/test_backtesting.py

@@ -840,7 +840,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
         'Downloading data for all pairs in whitelist ...',
-        'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...'
     ]
 
@@ -899,7 +899,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog):
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
         'Downloading data for all pairs in whitelist ...',
-        'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...',
         'Running backtesting for Strategy DefaultStrategy',
         'Running backtesting for Strategy TestStrategy',

freqtrade/tests/optimize/test_optimize.py

@@ -85,11 +85,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
     _clean_test_file(file)
 
 
-def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_conf) -> None:
+def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf) -> None:
     """
     Test load_data() with 1 min ticker
     """
-    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list)
     exchange = get_patched_exchange(mocker, default_conf)
     file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
 
@@ -119,8 +119,8 @@ def test_testdata_path() -> None:
     assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None)
 
 
-def test_download_pairs(ticker_history, mocker, default_conf) -> None:
+def test_download_pairs(ticker_history_list, mocker, default_conf) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list)
     exchange = get_patched_exchange(mocker, default_conf)
     file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
     file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
@@ -280,8 +280,8 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf)
     assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples)
 
 
-def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None:
+def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list)
     exchange = get_patched_exchange(mocker, default_conf)
     # Tst that pairs-cached is not touched.
     assert not exchange._pairs_last_refresh_time

freqtrade/tests/strategy/test_interface.py

@@ -16,62 +16,69 @@ from freqtrade.strategy.default_strategy import DefaultStrategy
 _STRATEGY = DefaultStrategy(config={})
 
 
-def test_returns_latest_buy_signal(mocker, default_conf):
+def test_returns_latest_buy_signal(mocker, default_conf, ticker_history):
     mocker.patch.object(
         _STRATEGY, 'analyze_ticker',
         return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}])
     )
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False)
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False)
 
     mocker.patch.object(
         _STRATEGY, 'analyze_ticker',
         return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}])
     )
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True)
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True)
 
 
-def test_returns_latest_sell_signal(mocker, default_conf):
+def test_returns_latest_sell_signal(mocker, default_conf, ticker_history):
     mocker.patch.object(
         _STRATEGY, 'analyze_ticker',
         return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}])
     )
 
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True)
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True)
 
     mocker.patch.object(
         _STRATEGY, 'analyze_ticker',
         return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}])
     )
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False)
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False)
 
 
 def test_get_signal_empty(default_conf, mocker, caplog):
     assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'],
-                                                  None)
+                                                  DataFrame())
     assert log_has('Empty ticker history for pair foo', caplog.record_tuples)
+    caplog.clear()
+
+    assert (False, False) == _STRATEGY.get_signal('bar', default_conf['ticker_interval'],
+                                                  [])
+    assert log_has('Empty ticker history for pair bar', caplog.record_tuples)
 
 
-def test_get_signal_exception_valueerror(default_conf, mocker, caplog):
+def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ticker_history):
     caplog.set_level(logging.INFO)
     mocker.patch.object(
         _STRATEGY, 'analyze_ticker',
         side_effect=ValueError('xyz')
     )
-    assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], 1)
+    assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'],
+                                                  ticker_history)
     assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples)
 
 
-def test_get_signal_empty_dataframe(default_conf, mocker, caplog):
+def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ticker_history):
     caplog.set_level(logging.INFO)
     mocker.patch.object(
         _STRATEGY, 'analyze_ticker',
         return_value=DataFrame([])
     )
-    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1)
+    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
+                                                  ticker_history)
     assert log_has('Empty dataframe for pair xyz', caplog.record_tuples)
 
 
-def test_get_signal_old_dataframe(default_conf, mocker, caplog):
+def test_get_signal_old_dataframe(default_conf, mocker, caplog, ticker_history):
     caplog.set_level(logging.INFO)
     # default_conf defines a 5m interval. we check interval * 2 + 5m
     # this is necessary as the last candle is removed (partial candles) by default
@@ -81,7 +88,8 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog):
         _STRATEGY, 'analyze_ticker',
         return_value=DataFrame(ticks)
     )
-    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1)
+    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
+                                                  ticker_history)
     assert log_has(
         'Outdated history for pair xyz. Last tick is 16 minutes old',
         caplog.record_tuples

freqtrade/tests/test_misc.py

@@ -16,8 +16,8 @@ def test_shorten_date() -> None:
     assert shorten_date(str_data) == str_shorten_data
 
 
-def test_datesarray_to_datetimearray(ticker_history):
+def test_datesarray_to_datetimearray(ticker_history_list):
-    dataframes = parse_ticker_dataframe(ticker_history)
+    dataframes = parse_ticker_dataframe(ticker_history_list)
     dates = datesarray_to_datetimearray(dataframes['date'])
 
     assert isinstance(dates[0], datetime.datetime)

scripts/plot_dataframe.py

@@ -139,7 +139,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
     if args.live:
         logger.info('Downloading pair.')
         exchange.refresh_tickers([pair], tick_interval)
-        tickers[pair] = exchange.klines[pair]
+        tickers[pair] = exchange.klines(pair)
     else:
         tickers = optimize.load_data(
             datadir=_CONF.get("datadir"),