Merge pull request #1915 from freqtrade/feat/drop_incomplete_optional

Make dropping the last candle optional (configured per exchange)

freqtrade/data/converter.py

@@ -10,14 +10,16 @@ from pandas import DataFrame, to_datetime
 logger = logging.getLogger(__name__)
 
 
-def parse_ticker_dataframe(ticker: list, ticker_interval: str,
-                           fill_missing: bool = True) -> DataFrame:
+def parse_ticker_dataframe(ticker: list, ticker_interval: str, *,
+                           fill_missing: bool = True,
+                           drop_incomplete: bool = True) -> DataFrame:
     """
     Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe
     :param ticker: ticker list, as returned by exchange.async_get_candle_history
     :param ticker_interval: ticker_interval (e.g. 5m). Used to fill up eventual missing data
     :param fill_missing: fill up missing candles with 0 candles
                          (see ohlcv_fill_up_missing_data for details)
+    :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete
     :return: DataFrame
     """
     logger.debug("Parsing tickerlist to dataframe")
@@ -43,8 +45,10 @@ def parse_ticker_dataframe(ticker: list, ticker_interval: str,
         'close': 'last',
         'volume': 'max',
     })
-    frame.drop(frame.tail(1).index, inplace=True)     # eliminate partial candle
-    logger.debug('Dropping last candle')
+    # eliminate partial candle
+    if drop_incomplete:
+        frame.drop(frame.tail(1).index, inplace=True)
+        logger.debug('Dropping last candle')
 
     if fill_missing:
         return ohlcv_fill_up_missing_data(frame, ticker_interval)

freqtrade/data/history.py

@@ -81,10 +81,20 @@ def load_pair_history(pair: str,
                       timerange: TimeRange = TimeRange(None, None, 0, 0),
                       refresh_pairs: bool = False,
                       exchange: Optional[Exchange] = None,
-                      fill_up_missing: bool = True
+                      fill_up_missing: bool = True,
+                      drop_incomplete: bool = True
                       ) -> DataFrame:
     """
     Loads cached ticker history for the given pair.
+    :param pair: Pair to load data for
+    :param ticker_interval: Ticker-interval (e.g. "5m")
+    :param datadir: Path to the data storage location.
+    :param timerange: Limit data to be loaded to this timerange
+    :param refresh_pairs: Refresh pairs from exchange.
+        (Note: Requires exchange to be passed as well.)
+    :param exchange: Exchange object (needed when using "refresh_pairs")
+    :param fill_up_missing: Fill missing values with "No action"-candles
+    :param drop_incomplete: Drop last candle assuming it may be incomplete.
     :return: DataFrame with ohlcv data
     """
 
@@ -106,7 +116,9 @@ def load_pair_history(pair: str,
             logger.warning('Missing data at end for pair %s, data ends at %s',
                            pair,
                            arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
-        return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing)
+        return parse_ticker_dataframe(pairdata, ticker_interval,
+                                      fill_missing=fill_up_missing,
+                                      drop_incomplete=drop_incomplete)
     else:
         logger.warning(
             f'No history data for pair: "{pair}", interval: {ticker_interval}. '

freqtrade/exchange/exchange.py

@@ -2,23 +2,24 @@
 """
 Cryptocurrency Exchanges support
 """
-import logging
+import asyncio
 import inspect
-from random import randint
-from typing import List, Dict, Tuple, Any, Optional
+import logging
+from copy import deepcopy
 from datetime import datetime
-from math import floor, ceil
+from math import ceil, floor
+from random import randint
+from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
-import asyncio
 import ccxt
 import ccxt.async_support as ccxt_async
 from pandas import DataFrame
 
-from freqtrade import (constants, DependencyException, OperationalException,
-                       TemporaryError, InvalidOrderException)
+from freqtrade import (DependencyException, InvalidOrderException,
+                       OperationalException, TemporaryError, constants)
 from freqtrade.data.converter import parse_ticker_dataframe
-
+from freqtrade.misc import deep_merge_dicts
 
 logger = logging.getLogger(__name__)
 
@@ -68,12 +69,15 @@ class Exchange(object):
     _params: Dict = {}
 
     # Dict to specify which options each exchange implements
-    # TODO: this should be merged with attributes from subclasses
-    # To avoid having to copy/paste this to all subclasses.
-    _ft_has: Dict = {
+    # This defines defaults, which can be selectively overridden by subclasses using _ft_has
+    # or by specifying them in the configuration.
+    _ft_has_default: Dict = {
         "stoploss_on_exchange": False,
         "order_time_in_force": ["gtc"],
+        "ohlcv_candle_limit": 500,
+        "ohlcv_partial_candle": True,
     }
+    _ft_has: Dict = {}
 
     def __init__(self, config: dict) -> None:
         """
@@ -100,6 +104,19 @@ class Exchange(object):
             logger.info('Instance is running with dry_run enabled')
 
         exchange_config = config['exchange']
+
+        # Deep merge ft_has with default ft_has options
+        self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default))
+        if exchange_config.get("_ft_has_params"):
+            self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"),
+                                            self._ft_has)
+            logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has)
+
+        # Assign this directly for easy access
+        self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
+        self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
+
+        # Initialize ccxt objects
         self._api: ccxt.Exchange = self._init_ccxt(
             exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config'))
         self._api_async: ccxt_async.Exchange = self._init_ccxt(
@@ -506,10 +523,8 @@ class Exchange(object):
     async def _async_get_history(self, pair: str,
                                  ticker_interval: str,
                                  since_ms: int) -> List:
-        # Assume exchange returns 500 candles
-        _LIMIT = 500
 
-        one_call = timeframe_to_msecs(ticker_interval) * _LIMIT
+        one_call = timeframe_to_msecs(ticker_interval) * self._ohlcv_candle_limit
         logger.debug(
             "one_call: %s msecs (%s)",
             one_call,
@@ -566,7 +581,8 @@ class Exchange(object):
                 self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000
             # keeping parsed dataframe in cache
             self._klines[(pair, ticker_interval)] = parse_ticker_dataframe(
-                ticks, ticker_interval, fill_missing=True)
+                ticks, ticker_interval, fill_missing=True,
+                drop_incomplete=self._ohlcv_partial_candle)
         return tickers
 
     def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool:

freqtrade/misc.py

@@ -117,6 +117,8 @@ def format_ms_time(date: int) -> str:
 
 def deep_merge_dicts(source, destination):
     """
+    Values from Source override destination, destination is returned (and modified!!)
+    Sample:
     >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } }
     >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } }
     >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } }

freqtrade/tests/conftest.py

@@ -649,7 +649,7 @@ def ticker_history_list():
 
 @pytest.fixture
 def ticker_history(ticker_history_list):
-    return parse_ticker_dataframe(ticker_history_list, "5m", True)
+    return parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True)
 
 
 @pytest.fixture
@@ -854,7 +854,7 @@ def tickers():
 @pytest.fixture
 def result():
     with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file:
-        return parse_ticker_dataframe(json.load(data_file), '1m', True)
+        return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True)
 
 # FIX:
 # Create an fixture/function

freqtrade/tests/data/test_converter.py

@@ -96,3 +96,50 @@ def test_ohlcv_fill_up_missing_data2(caplog):
 
     assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}",
                    caplog.record_tuples)
+
+
+def test_ohlcv_drop_incomplete(caplog):
+    ticker_interval = '1d'
+    ticks = [[
+            1559750400000,  # 2019-06-04
+            8.794e-05,  # open
+            8.948e-05,  # high
+            8.794e-05,  # low
+            8.88e-05,  # close
+            2255,  # volume (in quote currency)
+        ],
+        [
+            1559836800000,  # 2019-06-05
+            8.88e-05,
+            8.942e-05,
+            8.88e-05,
+            8.893e-05,
+            9911,
+        ],
+        [
+            1559923200000,  # 2019-06-06
+            8.891e-05,
+            8.893e-05,
+            8.875e-05,
+            8.877e-05,
+            2251
+        ],
+        [
+            1560009600000,  # 2019-06-07
+            8.877e-05,
+            8.883e-05,
+            8.895e-05,
+            8.817e-05,
+            123551
+     ]
+    ]
+    caplog.set_level(logging.DEBUG)
+    data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=False)
+    assert len(data) == 4
+    assert not log_has("Dropping last candle", caplog.record_tuples)
+
+    # Drop last candle
+    data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=True)
+    assert len(data) == 3
+
+    assert log_has("Dropping last candle", caplog.record_tuples)

freqtrade/tests/exchange/test_exchange.py

@@ -1435,3 +1435,30 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker):
     assert order['type'] == order_type
     assert order['price'] == 220
     assert order['amount'] == 1
+
+
+def test_merge_ft_has_dict(default_conf, mocker):
+    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock()))
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    ex = Exchange(default_conf)
+    assert ex._ft_has == Exchange._ft_has_default
+
+    ex = Kraken(default_conf)
+    assert ex._ft_has == Exchange._ft_has_default
+
+    # Binance defines different values
+    ex = Binance(default_conf)
+    assert ex._ft_has != Exchange._ft_has_default
+    assert ex._ft_has['stoploss_on_exchange']
+    assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
+
+    conf = copy.deepcopy(default_conf)
+    conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,
+                                          "stoploss_on_exchange": False}
+    # Use settings from configuration (overriding stoploss_on_exchange)
+    ex = Binance(conf)
+    assert ex._ft_has != Exchange._ft_has_default
+    assert not ex._ft_has['stoploss_on_exchange']
+    assert ex._ft_has['DeadBeef'] == 20

freqtrade/tests/strategy/test_interface.py

@@ -111,7 +111,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None:
 
     timerange = TimeRange(None, 'line', 0, -100)
     tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)}
+    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
     data = strategy.tickerdata_to_dataframe(tickerlist)
     assert len(data['UNITTEST/BTC']) == 102       # partial candle was removed