From d7c63347e12f41a99e3caaed498669f0230ea16e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 13:19:01 +0200 Subject: [PATCH 01/10] Use kwarg for parse_ticker_dataframe --- freqtrade/data/history.py | 2 +- freqtrade/tests/conftest.py | 4 ++-- freqtrade/tests/strategy/test_interface.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 2dacce8c6..72f7111f6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -106,7 +106,7 @@ def load_pair_history(pair: str, logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing) + return parse_ticker_dataframe(pairdata, ticker_interval, fill_missing=fill_up_missing) else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index a907b33ed..dcc69fcb1 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -649,7 +649,7 @@ def ticker_history_list(): @pytest.fixture def ticker_history(ticker_history_list): - return parse_ticker_dataframe(ticker_history_list, "5m", True) + return parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True) @pytest.fixture @@ -854,7 +854,7 @@ def tickers(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file: - return parse_ticker_dataframe(json.load(data_file), '1m', True) + return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True) # FIX: # Create an fixture/function diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index d6ef0c8e7..e384003dc 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -111,7 +111,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed From 9c497bf15c597921bb1c5509236f50414bbc4df4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:04:19 +0200 Subject: [PATCH 02/10] Improve docstring for deep_merge_dicts --- freqtrade/misc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 9d37214e4..460e20e91 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -117,6 +117,8 @@ def format_ms_time(date: int) -> str: def deep_merge_dicts(source, destination): """ + Values from Source override destination, destination is returned (and modified!!) + Sample: >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } } >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } } >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } } From 7108a2e57d557b4fc2580ec0defb2693c7e96164 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:05:36 +0200 Subject: [PATCH 03/10] Add deep_merge for _ft_has and test --- freqtrade/exchange/exchange.py | 31 +++++++++++++++-------- freqtrade/tests/exchange/test_exchange.py | 27 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 72a0efb1f..07a5e9037 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2,23 +2,24 @@ """ Cryptocurrency Exchanges support """ -import logging +import asyncio import inspect -from random import randint -from typing import List, Dict, Tuple, Any, Optional +import logging +from copy import deepcopy from datetime import datetime -from math import floor, ceil +from math import ceil, floor +from random import randint +from typing import Any, Dict, List, Optional, Tuple import arrow -import asyncio import ccxt import ccxt.async_support as ccxt_async from pandas import DataFrame -from freqtrade import (constants, DependencyException, OperationalException, - TemporaryError, InvalidOrderException) +from freqtrade import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError, constants) from freqtrade.data.converter import parse_ticker_dataframe - +from freqtrade.misc import deep_merge_dicts logger = logging.getLogger(__name__) @@ -68,12 +69,13 @@ class Exchange(object): _params: Dict = {} # Dict to specify which options each exchange implements - # TODO: this should be merged with attributes from subclasses - # To avoid having to copy/paste this to all subclasses. - _ft_has: Dict = { + # This defines defaults, which can be selectively overridden by subclasses using _ft_has + # or by specifying them in the configuration. + _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], } + _ft_has: Dict = {} def __init__(self, config: dict) -> None: """ @@ -100,6 +102,13 @@ class Exchange(object): logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] + + # Deep merge ft_has with default ft_has options + self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default)) + if exchange_config.get("_ft_has_params"): + self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"), + self._ft_has) + self._api: ccxt.Exchange = self._init_ccxt( exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) self._api_async: ccxt_async.Exchange = self._init_ccxt( diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index fda9c8241..f0dc96626 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1435,3 +1435,30 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker): assert order['type'] == order_type assert order['price'] == 220 assert order['amount'] == 1 + + +def test_merge_ft_has_dict(default_conf, mocker): + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + ex = Exchange(default_conf) + assert ex._ft_has == Exchange._ft_has_default + + ex = Kraken(default_conf) + assert ex._ft_has == Exchange._ft_has_default + + # Binance defines different values + ex = Binance(default_conf) + assert ex._ft_has != Exchange._ft_has_default + assert ex._ft_has['stoploss_on_exchange'] + assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc'] + + conf = copy.deepcopy(default_conf) + conf['exchange']['_ft_has_params'] = {"DeadBeef": 20, + "stoploss_on_exchange": False} + # Use settings from configuration (overriding stoploss_on_exchange) + ex = Binance(conf) + assert ex._ft_has != Exchange._ft_has_default + assert not ex._ft_has['stoploss_on_exchange'] + assert ex._ft_has['DeadBeef'] == 20 From 3fe5388d4cdce72a03f7420a5585eaf8edddbd21 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:13:03 +0200 Subject: [PATCH 04/10] Document _ft_has_params override --- docs/configuration.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index df116b3c2..bbadb87e8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -301,8 +301,24 @@ This configuration enables binance, as well as rate limiting to avoid bans from !!! Note Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. - We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. + We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. +#### Advanced FreqTrade Exchange configuration + +Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours. + +Available options are listed in the exchange-class as `_ft_has_default`. + +For example, to test the order type `FOK` with Kraken: + +```json +"exchange": { + "name": "kraken", + "_ft_has_params": {"order_time_in_force": ["gtc", "fok"]} +``` + +!!! Warning + Please make sure to fully understand the impacts of these settings before modifying them. ### What values can be used for fiat_display_currency? From fdbbefdddd8442f1439256b84ddb24f804e1490c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:35:58 +0200 Subject: [PATCH 05/10] Make drop_incomplete optional --- freqtrade/data/converter.py | 12 ++++++++---- freqtrade/exchange/exchange.py | 8 +++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 77a3447da..dc566070d 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -10,14 +10,16 @@ from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) -def parse_ticker_dataframe(ticker: list, ticker_interval: str, - fill_missing: bool = True) -> DataFrame: +def parse_ticker_dataframe(ticker: list, ticker_interval: str, *, + fill_missing: bool = True, + drop_incomplete: bool = True) -> DataFrame: """ Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe :param ticker: ticker list, as returned by exchange.async_get_candle_history :param ticker_interval: ticker_interval (e.g. 5m). Used to fill up eventual missing data :param fill_missing: fill up missing candles with 0 candles (see ohlcv_fill_up_missing_data for details) + :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete :return: DataFrame """ logger.debug("Parsing tickerlist to dataframe") @@ -43,8 +45,10 @@ def parse_ticker_dataframe(ticker: list, ticker_interval: str, 'close': 'last', 'volume': 'max', }) - frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle - logger.debug('Dropping last candle') + # eliminate partial candle + if drop_incomplete: + frame.drop(frame.tail(1).index, inplace=True) + logger.debug('Dropping last candle') if fill_missing: return ohlcv_fill_up_missing_data(frame, ticker_interval) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 07a5e9037..401a3571f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -74,6 +74,7 @@ class Exchange(object): _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "ohlcv_partial_candle": True, } _ft_has: Dict = {} @@ -108,7 +109,12 @@ class Exchange(object): if exchange_config.get("_ft_has_params"): self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"), self._ft_has) + logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) + # Assign this directly for easy access + self._drop_incomplete = self._ft_has['ohlcv_partial_candle'] + + # Initialize ccxt objects self._api: ccxt.Exchange = self._init_ccxt( exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) self._api_async: ccxt_async.Exchange = self._init_ccxt( @@ -575,7 +581,7 @@ class Exchange(object): self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, ticker_interval)] = parse_ticker_dataframe( - ticks, ticker_interval, fill_missing=True) + ticks, ticker_interval, fill_missing=True, drop_incomplete=self._drop_incomplete) return tickers def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool: From 6ad94684d501af503cf070edb714e21f54ce2886 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:36:08 +0200 Subject: [PATCH 06/10] Add WIP document of steps to test a new exchange --- docs/developer.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index e7f79bc1c..ccd5cdd8f 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -81,6 +81,50 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs This is a simple method used by `VolumePairList` - however serves as a good example. It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider. +## Implement a new Exchange (WIP) + +!!! Note + This section is a Work in Progress and is not a complete guide on how to test a new exchange with FreqTrade. + +Most exchanges supported by CCXT should work out of the box. + +### Stoploss On Exchange + +Check if the new exchange supports Stoploss on Exchange orders through their API. + +Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects. + +### Incomplete candles + +While fetching OHLCV data, we're may end up getting incomplete candles (Depending on the exchange). +To demonstrate this, we'll use daily candles (`"1d"`) to keep things simple. +We query the api (`ct.fetch_ohlcv()`) for the timeframe and look at the date of the last entry. If this entry changes or shows the date of a "incomplete" candle, then we should drop this since having incomplete candles is problematic because indicators assume that only complete candles are passed to them, and will generate a lot of false buy signals. By default, we're therefore removing the last candle assuming it's incomplete. + +To check how the new exchange behaves, you can use the following snippet: + +``` python +import ccxt +from datetime import datetime +from freqtrade.data.converter import parse_ticker_dataframe +ct = ccxt.binance() +timeframe = "1d" +raw = ct.fetch_ohlcv(pair, timeframe=timeframe) + +# convert to dataframe +df1 = parse_ticker_dataframe(raw, timeframe, drop_incomplete=False) + +print(df1["date"].tail(1)) +print(datetime.utcnow()) +``` + +``` output +19 2019-06-08 00:00:00+00:00 +2019-06-09 12:30:27.873327 +``` + +The output will show the last entry from the Exchange as well as the current UTC date. +If the day shows the same day, then the last candle can be assumed as incomplete and should be dropped (leave the setting `"ohlcv_partial_candle"` from the exchange-class untouched / True). Otherwise, set `"ohlcv_partial_candle"` to `False` to not drop Candles (shown in the example above). + ## Creating a release This part of the documentation is aimed at maintainers, and shows how to create a release. From ce317b62f9b3d523ccaa9e8474f99be2c36966a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:40:45 +0200 Subject: [PATCH 07/10] Add docstrings to load_pair_history --- freqtrade/data/history.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 72f7111f6..67f942119 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -81,10 +81,20 @@ def load_pair_history(pair: str, timerange: TimeRange = TimeRange(None, None, 0, 0), refresh_pairs: bool = False, exchange: Optional[Exchange] = None, - fill_up_missing: bool = True + fill_up_missing: bool = True, + drop_incomplete: bool = True ) -> DataFrame: """ Loads cached ticker history for the given pair. + :param pair: Pair to load data for + :param ticker_interval: Ticker-interval (e.g. "5m") + :param datadir: Path to the data storage location. + :param timerange: Limit data to be loaded to this timerange + :param refresh_pairs: Refresh pairs from exchange. + (Note: Requires exchange to be passed as well.) + :param exchange: Exchange object (needed when using "refresh_pairs") + :param fill_up_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. :return: DataFrame with ohlcv data """ @@ -106,7 +116,9 @@ def load_pair_history(pair: str, logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - return parse_ticker_dataframe(pairdata, ticker_interval, fill_missing=fill_up_missing) + return parse_ticker_dataframe(pairdata, ticker_interval, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete) else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' From 3380543878f664089038357d7554d947cafcfd4e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:51:58 +0200 Subject: [PATCH 08/10] Add test for drop_incomplete option --- freqtrade/tests/data/test_converter.py | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/freqtrade/tests/data/test_converter.py b/freqtrade/tests/data/test_converter.py index 4c8de575d..8a0761f1c 100644 --- a/freqtrade/tests/data/test_converter.py +++ b/freqtrade/tests/data/test_converter.py @@ -96,3 +96,50 @@ def test_ohlcv_fill_up_missing_data2(caplog): assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}", caplog.record_tuples) + + +def test_ohlcv_drop_incomplete(caplog): + ticker_interval = '1d' + ticks = [[ + 1559750400000, # 2019-06-04 + 8.794e-05, # open + 8.948e-05, # high + 8.794e-05, # low + 8.88e-05, # close + 2255, # volume (in quote currency) + ], + [ + 1559836800000, # 2019-06-05 + 8.88e-05, + 8.942e-05, + 8.88e-05, + 8.893e-05, + 9911, + ], + [ + 1559923200000, # 2019-06-06 + 8.891e-05, + 8.893e-05, + 8.875e-05, + 8.877e-05, + 2251 + ], + [ + 1560009600000, # 2019-06-07 + 8.877e-05, + 8.883e-05, + 8.895e-05, + 8.817e-05, + 123551 + ] + ] + caplog.set_level(logging.DEBUG) + data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=False) + assert len(data) == 4 + assert not log_has("Dropping last candle", caplog.record_tuples) + + # Drop last candle + data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=True) + assert len(data) == 3 + + assert log_has("Dropping last candle", caplog.record_tuples) From 9f2e0b11d19772cfbe165ad6f8e67099ab98c175 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:52:17 +0200 Subject: [PATCH 09/10] Parametrize ohlcv_candle_limit (per call) --- docs/configuration.md | 7 +++++-- freqtrade/exchange/exchange.py | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bbadb87e8..98953d73f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -309,12 +309,15 @@ Advanced options can be configured using the `_ft_has_params` setting, which wil Available options are listed in the exchange-class as `_ft_has_default`. -For example, to test the order type `FOK` with Kraken: +For example, to test the order type `FOK` with Kraken, and modify candle_limit to 200 (so you only get 200 candles per call): ```json "exchange": { "name": "kraken", - "_ft_has_params": {"order_time_in_force": ["gtc", "fok"]} + "_ft_has_params": { + "order_time_in_force": ["gtc", "fok"], + "ohlcv_candle_limit": 200 + } ``` !!! Warning diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 401a3571f..ea6996efb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -74,6 +74,7 @@ class Exchange(object): _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, } _ft_has: Dict = {} @@ -112,7 +113,8 @@ class Exchange(object): logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) # Assign this directly for easy access - self._drop_incomplete = self._ft_has['ohlcv_partial_candle'] + self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit'] + self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle'] # Initialize ccxt objects self._api: ccxt.Exchange = self._init_ccxt( @@ -521,10 +523,8 @@ class Exchange(object): async def _async_get_history(self, pair: str, ticker_interval: str, since_ms: int) -> List: - # Assume exchange returns 500 candles - _LIMIT = 500 - one_call = timeframe_to_msecs(ticker_interval) * _LIMIT + one_call = timeframe_to_msecs(ticker_interval) * self._ohlcv_candle_limit logger.debug( "one_call: %s msecs (%s)", one_call, @@ -581,7 +581,8 @@ class Exchange(object): self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, ticker_interval)] = parse_ticker_dataframe( - ticks, ticker_interval, fill_missing=True, drop_incomplete=self._drop_incomplete) + ticks, ticker_interval, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) return tickers def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool: From 792390e8159a7f6f6e211f0a10146de4cc3741f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 15:03:26 +0200 Subject: [PATCH 10/10] Add missing parameter for exchange-verify snippet --- docs/developer.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/developer.md b/docs/developer.md index ccd5cdd8f..6ecb7f156 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -108,6 +108,7 @@ from datetime import datetime from freqtrade.data.converter import parse_ticker_dataframe ct = ccxt.binance() timeframe = "1d" +pair = "XLM/BTC" # Make sure to use a pair that exists on that exchange! raw = ct.fetch_ohlcv(pair, timeframe=timeframe) # convert to dataframe