diff --git a/docs/configuration.md b/docs/configuration.md index df116b3c2..98953d73f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -301,8 +301,27 @@ This configuration enables binance, as well as rate limiting to avoid bans from !!! Note Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. - We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. + We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. +#### Advanced FreqTrade Exchange configuration + +Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours. + +Available options are listed in the exchange-class as `_ft_has_default`. + +For example, to test the order type `FOK` with Kraken, and modify candle_limit to 200 (so you only get 200 candles per call): + +```json +"exchange": { + "name": "kraken", + "_ft_has_params": { + "order_time_in_force": ["gtc", "fok"], + "ohlcv_candle_limit": 200 + } +``` + +!!! Warning + Please make sure to fully understand the impacts of these settings before modifying them. ### What values can be used for fiat_display_currency? diff --git a/docs/developer.md b/docs/developer.md index e7f79bc1c..6ecb7f156 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -81,6 +81,51 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs This is a simple method used by `VolumePairList` - however serves as a good example. It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider. +## Implement a new Exchange (WIP) + +!!! Note + This section is a Work in Progress and is not a complete guide on how to test a new exchange with FreqTrade. + +Most exchanges supported by CCXT should work out of the box. + +### Stoploss On Exchange + +Check if the new exchange supports Stoploss on Exchange orders through their API. + +Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects. + +### Incomplete candles + +While fetching OHLCV data, we're may end up getting incomplete candles (Depending on the exchange). +To demonstrate this, we'll use daily candles (`"1d"`) to keep things simple. +We query the api (`ct.fetch_ohlcv()`) for the timeframe and look at the date of the last entry. If this entry changes or shows the date of a "incomplete" candle, then we should drop this since having incomplete candles is problematic because indicators assume that only complete candles are passed to them, and will generate a lot of false buy signals. By default, we're therefore removing the last candle assuming it's incomplete. + +To check how the new exchange behaves, you can use the following snippet: + +``` python +import ccxt +from datetime import datetime +from freqtrade.data.converter import parse_ticker_dataframe +ct = ccxt.binance() +timeframe = "1d" +pair = "XLM/BTC" # Make sure to use a pair that exists on that exchange! +raw = ct.fetch_ohlcv(pair, timeframe=timeframe) + +# convert to dataframe +df1 = parse_ticker_dataframe(raw, timeframe, drop_incomplete=False) + +print(df1["date"].tail(1)) +print(datetime.utcnow()) +``` + +``` output +19 2019-06-08 00:00:00+00:00 +2019-06-09 12:30:27.873327 +``` + +The output will show the last entry from the Exchange as well as the current UTC date. +If the day shows the same day, then the last candle can be assumed as incomplete and should be dropped (leave the setting `"ohlcv_partial_candle"` from the exchange-class untouched / True). Otherwise, set `"ohlcv_partial_candle"` to `False` to not drop Candles (shown in the example above). + ## Creating a release This part of the documentation is aimed at maintainers, and shows how to create a release. diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 77a3447da..dc566070d 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -10,14 +10,16 @@ from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) -def parse_ticker_dataframe(ticker: list, ticker_interval: str, - fill_missing: bool = True) -> DataFrame: +def parse_ticker_dataframe(ticker: list, ticker_interval: str, *, + fill_missing: bool = True, + drop_incomplete: bool = True) -> DataFrame: """ Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe :param ticker: ticker list, as returned by exchange.async_get_candle_history :param ticker_interval: ticker_interval (e.g. 5m). Used to fill up eventual missing data :param fill_missing: fill up missing candles with 0 candles (see ohlcv_fill_up_missing_data for details) + :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete :return: DataFrame """ logger.debug("Parsing tickerlist to dataframe") @@ -43,8 +45,10 @@ def parse_ticker_dataframe(ticker: list, ticker_interval: str, 'close': 'last', 'volume': 'max', }) - frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle - logger.debug('Dropping last candle') + # eliminate partial candle + if drop_incomplete: + frame.drop(frame.tail(1).index, inplace=True) + logger.debug('Dropping last candle') if fill_missing: return ohlcv_fill_up_missing_data(frame, ticker_interval) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 2dacce8c6..67f942119 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -81,10 +81,20 @@ def load_pair_history(pair: str, timerange: TimeRange = TimeRange(None, None, 0, 0), refresh_pairs: bool = False, exchange: Optional[Exchange] = None, - fill_up_missing: bool = True + fill_up_missing: bool = True, + drop_incomplete: bool = True ) -> DataFrame: """ Loads cached ticker history for the given pair. + :param pair: Pair to load data for + :param ticker_interval: Ticker-interval (e.g. "5m") + :param datadir: Path to the data storage location. + :param timerange: Limit data to be loaded to this timerange + :param refresh_pairs: Refresh pairs from exchange. + (Note: Requires exchange to be passed as well.) + :param exchange: Exchange object (needed when using "refresh_pairs") + :param fill_up_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. :return: DataFrame with ohlcv data """ @@ -106,7 +116,9 @@ def load_pair_history(pair: str, logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing) + return parse_ticker_dataframe(pairdata, ticker_interval, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete) else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 72a0efb1f..ea6996efb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2,23 +2,24 @@ """ Cryptocurrency Exchanges support """ -import logging +import asyncio import inspect -from random import randint -from typing import List, Dict, Tuple, Any, Optional +import logging +from copy import deepcopy from datetime import datetime -from math import floor, ceil +from math import ceil, floor +from random import randint +from typing import Any, Dict, List, Optional, Tuple import arrow -import asyncio import ccxt import ccxt.async_support as ccxt_async from pandas import DataFrame -from freqtrade import (constants, DependencyException, OperationalException, - TemporaryError, InvalidOrderException) +from freqtrade import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError, constants) from freqtrade.data.converter import parse_ticker_dataframe - +from freqtrade.misc import deep_merge_dicts logger = logging.getLogger(__name__) @@ -68,12 +69,15 @@ class Exchange(object): _params: Dict = {} # Dict to specify which options each exchange implements - # TODO: this should be merged with attributes from subclasses - # To avoid having to copy/paste this to all subclasses. - _ft_has: Dict = { + # This defines defaults, which can be selectively overridden by subclasses using _ft_has + # or by specifying them in the configuration. + _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "ohlcv_candle_limit": 500, + "ohlcv_partial_candle": True, } + _ft_has: Dict = {} def __init__(self, config: dict) -> None: """ @@ -100,6 +104,19 @@ class Exchange(object): logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] + + # Deep merge ft_has with default ft_has options + self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default)) + if exchange_config.get("_ft_has_params"): + self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"), + self._ft_has) + logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) + + # Assign this directly for easy access + self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit'] + self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle'] + + # Initialize ccxt objects self._api: ccxt.Exchange = self._init_ccxt( exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) self._api_async: ccxt_async.Exchange = self._init_ccxt( @@ -506,10 +523,8 @@ class Exchange(object): async def _async_get_history(self, pair: str, ticker_interval: str, since_ms: int) -> List: - # Assume exchange returns 500 candles - _LIMIT = 500 - one_call = timeframe_to_msecs(ticker_interval) * _LIMIT + one_call = timeframe_to_msecs(ticker_interval) * self._ohlcv_candle_limit logger.debug( "one_call: %s msecs (%s)", one_call, @@ -566,7 +581,8 @@ class Exchange(object): self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, ticker_interval)] = parse_ticker_dataframe( - ticks, ticker_interval, fill_missing=True) + ticks, ticker_interval, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) return tickers def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 9d37214e4..460e20e91 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -117,6 +117,8 @@ def format_ms_time(date: int) -> str: def deep_merge_dicts(source, destination): """ + Values from Source override destination, destination is returned (and modified!!) + Sample: >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } } >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } } >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } } diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index a907b33ed..dcc69fcb1 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -649,7 +649,7 @@ def ticker_history_list(): @pytest.fixture def ticker_history(ticker_history_list): - return parse_ticker_dataframe(ticker_history_list, "5m", True) + return parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True) @pytest.fixture @@ -854,7 +854,7 @@ def tickers(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file: - return parse_ticker_dataframe(json.load(data_file), '1m', True) + return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True) # FIX: # Create an fixture/function diff --git a/freqtrade/tests/data/test_converter.py b/freqtrade/tests/data/test_converter.py index 4c8de575d..8a0761f1c 100644 --- a/freqtrade/tests/data/test_converter.py +++ b/freqtrade/tests/data/test_converter.py @@ -96,3 +96,50 @@ def test_ohlcv_fill_up_missing_data2(caplog): assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}", caplog.record_tuples) + + +def test_ohlcv_drop_incomplete(caplog): + ticker_interval = '1d' + ticks = [[ + 1559750400000, # 2019-06-04 + 8.794e-05, # open + 8.948e-05, # high + 8.794e-05, # low + 8.88e-05, # close + 2255, # volume (in quote currency) + ], + [ + 1559836800000, # 2019-06-05 + 8.88e-05, + 8.942e-05, + 8.88e-05, + 8.893e-05, + 9911, + ], + [ + 1559923200000, # 2019-06-06 + 8.891e-05, + 8.893e-05, + 8.875e-05, + 8.877e-05, + 2251 + ], + [ + 1560009600000, # 2019-06-07 + 8.877e-05, + 8.883e-05, + 8.895e-05, + 8.817e-05, + 123551 + ] + ] + caplog.set_level(logging.DEBUG) + data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=False) + assert len(data) == 4 + assert not log_has("Dropping last candle", caplog.record_tuples) + + # Drop last candle + data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=True) + assert len(data) == 3 + + assert log_has("Dropping last candle", caplog.record_tuples) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index fda9c8241..f0dc96626 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1435,3 +1435,30 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker): assert order['type'] == order_type assert order['price'] == 220 assert order['amount'] == 1 + + +def test_merge_ft_has_dict(default_conf, mocker): + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + ex = Exchange(default_conf) + assert ex._ft_has == Exchange._ft_has_default + + ex = Kraken(default_conf) + assert ex._ft_has == Exchange._ft_has_default + + # Binance defines different values + ex = Binance(default_conf) + assert ex._ft_has != Exchange._ft_has_default + assert ex._ft_has['stoploss_on_exchange'] + assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc'] + + conf = copy.deepcopy(default_conf) + conf['exchange']['_ft_has_params'] = {"DeadBeef": 20, + "stoploss_on_exchange": False} + # Use settings from configuration (overriding stoploss_on_exchange) + ex = Binance(conf) + assert ex._ft_has != Exchange._ft_has_default + assert not ex._ft_has['stoploss_on_exchange'] + assert ex._ft_has['DeadBeef'] == 20 diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index d6ef0c8e7..e384003dc 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -111,7 +111,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed