From a8f523adae66fb0eb4f28ffff4cd8394c7b6b0d4 Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Mon, 11 May 2020 19:32:28 +0400 Subject: [PATCH 1/7] attached pairlist manager onto dataprovider init for unified access to dynamic whitelist --- freqtrade/data/dataprovider.py | 12 +++++---- freqtrade/freqtradebot.py | 6 ++--- tests/data/test_dataprovider.py | 46 ++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index af0914939..01397d6b7 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode @@ -18,9 +19,10 @@ logger = logging.getLogger(__name__) class DataProvider: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange + self._pairlists: Optional = pairlists def refresh(self, pairlist: List[Tuple[str, str]], @@ -125,8 +127,8 @@ class DataProvider: As available pairs does not show whitelist until after informative pairs have been cached. :return: list of pairs in whitelist """ - from freqtrade.pairlist.pairlistmanager import PairListManager - pairlists = PairListManager(self._exchange, self._config) - pairlists.refresh_pairlist() - return pairlists.whitelist + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException("Dataprovider was not initialized with a pairlist provider.") diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4f4b3e3bb..73f0873e4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -71,15 +71,15 @@ class FreqtradeBot: self.wallets = Wallets(self.config, self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + self.pairlists = PairListManager(self.exchange, self.config) + + self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - self.pairlists = PairListManager(self.exchange, self.config) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index e5af80bc8..247703619 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,10 +1,12 @@ -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock, patch from pandas import DataFrame +import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from tests.conftest import get_patched_exchange, get_patched_freqtradebot +from tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ohlcv_history): @@ -151,21 +153,29 @@ def test_market(mocker, default_conf, markets): assert res is None -def test_current_whitelist(mocker, shitcoinmarkets, tickers, default_conf): - default_conf.update( - {"pairlists": [{"method": "VolumePairList", - "number_assets": 10, - "sort_key": "quoteVolume"}], }, ) - default_conf['exchange']['pair_blacklist'] = ['BLK/BTC'] +@patch('freqtrade.pairlist.pairlistmanager.PairListManager') +@patch('freqtrade.exchange.Exchange') +def test_current_whitelist(exchange, PairListManager, default_conf): + # patch default conf to volumepairlist + default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} - mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, - exchange_has=MagicMock(return_value=True), ) - bot = get_patched_freqtradebot(mocker, default_conf) - # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=shitcoinmarkets), ) - # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC', 'FUEL/BTC'] + pairlist = PairListManager(exchange, default_conf) + dp = DataProvider(default_conf, exchange, pairlist) - current_wl = bot.dataprovider.current_whitelist() - assert whitelist == current_wl + # Simulate volumepairs from exchange. + # pairlist.refresh_pairlist() + # Set the pairs manually... this would be done in refresh pairlist default whitelist + volumePL - blacklist + default_whitelist = default_conf['exchange']['pair_whitelist'] + default_blacklist = default_conf['exchange']['pair_blacklist'] + volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] + current_whitelist = list(set(volume_pairlist + default_whitelist)) + for pair in default_blacklist: + if pair in current_whitelist: + current_whitelist.remove(pair) + pairlist._whitelist = current_whitelist + + assert dp.current_whitelist() == pairlist._whitelist + + with pytest.raises(OperationalException) as e: + dp = DataProvider(default_conf, exchange) + dp.current_whitelist() From bc9efc31ad6c6e0806111384e6fa8229285d298a Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Wed, 6 May 2020 19:48:57 +0400 Subject: [PATCH 2/7] Added Method for accessing current pair list on initialization for dynamic informative pairs moved import into function to avoid circular import with hyperopt --- freqtrade/data/dataprovider.py | 14 ++++++++++++++ tests/data/test_dataprovider.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 1df710152..af0914939 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -116,3 +116,17 @@ class DataProvider: can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". """ return RunMode(self._config.get('runmode', RunMode.OTHER)) + + def current_whitelist(self) -> List[str]: + """ + fetch latest available whitelist. + + Useful when you have a large whitelist and need to call each pair as an informative pair. + As available pairs does not show whitelist until after informative pairs have been cached. + :return: list of pairs in whitelist + """ + from freqtrade.pairlist.pairlistmanager import PairListManager + + pairlists = PairListManager(self._exchange, self._config) + pairlists.refresh_pairlist() + return pairlists.whitelist diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 2b3dda188..e5af80bc8 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,10 +1,10 @@ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider from freqtrade.state import RunMode -from tests.conftest import get_patched_exchange +from tests.conftest import get_patched_exchange, get_patched_freqtradebot def test_ohlcv(mocker, default_conf, ohlcv_history): @@ -64,8 +64,8 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history): assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty # Test with and without parameter - assert dp.get_pair_dataframe("UNITTEST/BTC", - ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC")) + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\ + .equals(dp.get_pair_dataframe("UNITTEST/BTC")) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) @@ -90,10 +90,7 @@ def test_available_pairs(mocker, default_conf, ohlcv_history): dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == [ - ("XRP/BTC", ticker_interval), - ("UNITTEST/BTC", ticker_interval), - ] + assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ] def test_refresh(mocker, default_conf, ohlcv_history): @@ -152,3 +149,23 @@ def test_market(mocker, default_conf, markets): res = dp.market('UNITTEST/BTC') assert res is None + + +def test_current_whitelist(mocker, shitcoinmarkets, tickers, default_conf): + default_conf.update( + {"pairlists": [{"method": "VolumePairList", + "number_assets": 10, + "sort_key": "quoteVolume"}], }, ) + default_conf['exchange']['pair_blacklist'] = ['BLK/BTC'] + + mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, + exchange_has=MagicMock(return_value=True), ) + bot = get_patched_freqtradebot(mocker, default_conf) + # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=shitcoinmarkets), ) + # argument: use the whitelist dynamically by exchange-volume + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC', 'FUEL/BTC'] + + current_wl = bot.dataprovider.current_whitelist() + assert whitelist == current_wl From a5bfa5515cbdbab500dbe9df614730107804bacd Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Mon, 11 May 2020 20:13:06 +0400 Subject: [PATCH 3/7] Fix flake8 mypy --- freqtrade/data/dataprovider.py | 2 +- tests/data/test_dataprovider.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 01397d6b7..984652e24 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -22,7 +22,7 @@ class DataProvider: def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange - self._pairlists: Optional = pairlists + self._pairlists = pairlists def refresh(self, pairlist: List[Tuple[str, str]], diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 247703619..45ce1c009 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -164,7 +164,9 @@ def test_current_whitelist(exchange, PairListManager, default_conf): # Simulate volumepairs from exchange. # pairlist.refresh_pairlist() - # Set the pairs manually... this would be done in refresh pairlist default whitelist + volumePL - blacklist + + # Set the pairs manually... this would be done in refresh pairlist + # default whitelist + volumePL - blacklist default_whitelist = default_conf['exchange']['pair_whitelist'] default_blacklist = default_conf['exchange']['pair_blacklist'] volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] @@ -176,6 +178,6 @@ def test_current_whitelist(exchange, PairListManager, default_conf): assert dp.current_whitelist() == pairlist._whitelist - with pytest.raises(OperationalException) as e: + with pytest.raises(OperationalException): dp = DataProvider(default_conf, exchange) dp.current_whitelist() From 9fbe1357902aa90243309781f164e7328fdc5b9a Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Mon, 11 May 2020 19:32:28 +0400 Subject: [PATCH 4/7] attached pairlist manager onto dataprovider init for unified access to dynamic whitelist --- freqtrade/data/dataprovider.py | 12 +++++---- freqtrade/freqtradebot.py | 6 ++--- tests/data/test_dataprovider.py | 48 ++++++++++++++++++++------------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index af0914939..984652e24 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode @@ -18,9 +19,10 @@ logger = logging.getLogger(__name__) class DataProvider: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange + self._pairlists = pairlists def refresh(self, pairlist: List[Tuple[str, str]], @@ -125,8 +127,8 @@ class DataProvider: As available pairs does not show whitelist until after informative pairs have been cached. :return: list of pairs in whitelist """ - from freqtrade.pairlist.pairlistmanager import PairListManager - pairlists = PairListManager(self._exchange, self._config) - pairlists.refresh_pairlist() - return pairlists.whitelist + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException("Dataprovider was not initialized with a pairlist provider.") diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4f4b3e3bb..73f0873e4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -71,15 +71,15 @@ class FreqtradeBot: self.wallets = Wallets(self.config, self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + self.pairlists = PairListManager(self.exchange, self.config) + + self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - self.pairlists = PairListManager(self.exchange, self.config) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index e5af80bc8..45ce1c009 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,10 +1,12 @@ -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock, patch from pandas import DataFrame +import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from tests.conftest import get_patched_exchange, get_patched_freqtradebot +from tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ohlcv_history): @@ -151,21 +153,31 @@ def test_market(mocker, default_conf, markets): assert res is None -def test_current_whitelist(mocker, shitcoinmarkets, tickers, default_conf): - default_conf.update( - {"pairlists": [{"method": "VolumePairList", - "number_assets": 10, - "sort_key": "quoteVolume"}], }, ) - default_conf['exchange']['pair_blacklist'] = ['BLK/BTC'] +@patch('freqtrade.pairlist.pairlistmanager.PairListManager') +@patch('freqtrade.exchange.Exchange') +def test_current_whitelist(exchange, PairListManager, default_conf): + # patch default conf to volumepairlist + default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} - mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, - exchange_has=MagicMock(return_value=True), ) - bot = get_patched_freqtradebot(mocker, default_conf) - # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=shitcoinmarkets), ) - # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC', 'FUEL/BTC'] + pairlist = PairListManager(exchange, default_conf) + dp = DataProvider(default_conf, exchange, pairlist) - current_wl = bot.dataprovider.current_whitelist() - assert whitelist == current_wl + # Simulate volumepairs from exchange. + # pairlist.refresh_pairlist() + + # Set the pairs manually... this would be done in refresh pairlist + # default whitelist + volumePL - blacklist + default_whitelist = default_conf['exchange']['pair_whitelist'] + default_blacklist = default_conf['exchange']['pair_blacklist'] + volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] + current_whitelist = list(set(volume_pairlist + default_whitelist)) + for pair in default_blacklist: + if pair in current_whitelist: + current_whitelist.remove(pair) + pairlist._whitelist = current_whitelist + + assert dp.current_whitelist() == pairlist._whitelist + + with pytest.raises(OperationalException): + dp = DataProvider(default_conf, exchange) + dp.current_whitelist() From e864db1843c378b0eb8c6936f1b8a4aa3230eddc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 May 2020 06:38:14 +0200 Subject: [PATCH 5/7] Update test for dp.current_whitelist --- tests/data/test_dataprovider.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 45ce1c009..3e42abb95 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,9 +1,10 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from pandas import DataFrame import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from tests.conftest import get_patched_exchange @@ -153,30 +154,24 @@ def test_market(mocker, default_conf, markets): assert res is None -@patch('freqtrade.pairlist.pairlistmanager.PairListManager') -@patch('freqtrade.exchange.Exchange') -def test_current_whitelist(exchange, PairListManager, default_conf): +def test_current_whitelist(mocker, default_conf, tickers): # patch default conf to volumepairlist default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + get_tickers=tickers) + exchange = get_patched_exchange(mocker, default_conf) + pairlist = PairListManager(exchange, default_conf) dp = DataProvider(default_conf, exchange, pairlist) # Simulate volumepairs from exchange. - # pairlist.refresh_pairlist() - - # Set the pairs manually... this would be done in refresh pairlist - # default whitelist + volumePL - blacklist - default_whitelist = default_conf['exchange']['pair_whitelist'] - default_blacklist = default_conf['exchange']['pair_blacklist'] - volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] - current_whitelist = list(set(volume_pairlist + default_whitelist)) - for pair in default_blacklist: - if pair in current_whitelist: - current_whitelist.remove(pair) - pairlist._whitelist = current_whitelist + pairlist.refresh_pairlist() assert dp.current_whitelist() == pairlist._whitelist + # The identity of the 2 lists should be identical + assert dp.current_whitelist() is pairlist._whitelist with pytest.raises(OperationalException): dp = DataProvider(default_conf, exchange) From 63dfe3669ff994b535adde5f1609bbba283de69f Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Wed, 13 May 2020 00:25:57 +0400 Subject: [PATCH 6/7] Updated docs for #3267 --- docs/strategy-customization.md | 160 +++++++++++++++++++++------------ 1 file changed, 105 insertions(+), 55 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index c4fc55811..20cde7556 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -324,62 +324,9 @@ class Awesomestrategy(IStrategy): !!! Note If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. -### Additional data (DataProvider) +*** -The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. - -All methods return `None` in case of failure (do not raise an exception). - -Please always check the mode of operation to select the correct method to get data (samples see below). - -#### Possible options for DataProvider - -- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval). -- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. -- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. -- `get_pair_dataframe(pair, timeframe)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). -- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. -- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. -- `runmode` - Property containing the current runmode. - -#### Example: fetch live / historical candle (OHLCV) data for the first informative pair - -``` python -if self.dp: - inf_pair, inf_timeframe = self.informative_pairs()[0] - informative = self.dp.get_pair_dataframe(pair=inf_pair, - timeframe=inf_timeframe) -``` - -!!! Warning "Warning about backtesting" - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` - for the backtesting runmode) provides the full time-range in one go, - so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). - -!!! Warning "Warning in hyperopt" - This option cannot currently be used during hyperopt. - -#### Orderbook - -``` python -if self.dp: - if self.dp.runmode.value in ('live', 'dry_run'): - ob = self.dp.orderbook(metadata['pair'], 1) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] -``` - -!!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work if this - method is used. - -#### Available Pairs - -``` python -if self.dp: - for pair, timeframe in self.dp.available_pairs: - print(f"available {pair}, {timeframe}") -``` +### Additional data (informative_pairs) #### Get data for non-tradeable pairs @@ -404,6 +351,108 @@ def informative_pairs(self): It is however better to use resampling to longer time-intervals when possible to avoid hammering the exchange with too many requests and risk being blocked. +*** + +### Additional data (DataProvider) + +The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. + +All methods return `None` in case of failure (do not raise an exception). + +Please always check the mode of operation to select the correct method to get data (samples see below). + +#### Possible options for DataProvider + +- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval). +- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist) +- [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). +- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. +- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. +- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. +- [`orderbook(pair, maximum)`](#orderbookpair-maximum) - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. +- `runmode` - Property containing the current runmode. + +#### Example Usages: + +#### *available_pairs* + +``` python +if self.dp: + for pair, timeframe in self.dp.available_pairs: + print(f"available {pair}, {timeframe}") +``` + +#### *current_whitelist()* +Imagine you've developed a strategy that trades the `1m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. + +The strategy might look something like this: + +*Scan through the top 10 pairs by volume using the `VolumePairList` every minute and use a 14 day ATR to buy and sell.* + +Due to the limited available data, it's impossible to resample our `1m` candles into daily candles for use in the 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1/3 of a daily candle. We need 14 days at least! + +Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. + +This is where calling `self.dp.current_whitelist()` comes in handy. + +```python +class SampleStrategy(IStrategy): + # strategy init stuff... + + ticker_interval = '1m' + + # more strategy init stuff.. + + def informative_pairs(self): + + # get access to all pairs available in whitelist. + pairs = self.dp.current_whitelist() + # Assign tf to each pair so they can be downloaded and cached for strategy. + informative_pairs = [(pair, '1d') for pair in pairs] + return informative_pairs + + def populate_indicators(self, dataframe, metadata): + # Get the informative pair + informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') + # Get the 14 day ATR. + atr = ta.ATR(informative, timeperiod=14) + # Assign the Daily atr to the 1 minute dataframe. + dataframe['daily_atr'] = atr +``` + +#### *get_pair_dataframe(pair, timeframe)* + +``` python +# fetch live / historical candle (OHLCV) data for the first informative pair +if self.dp: + inf_pair, inf_timeframe = self.informative_pairs()[0] + informative = self.dp.get_pair_dataframe(pair=inf_pair, + timeframe=inf_timeframe) +``` + +!!! Warning "Warning about backtesting" + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + for the backtesting runmode) provides the full time-range in one go, + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + +!!! Warning "Warning in hyperopt" + This option cannot currently be used during hyperopt. + +#### *orderbook(pair, maximum)* + +``` python +if self.dp: + if self.dp.runmode.value in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] +``` + +!!! Warning + The order book is not part of the historic data which means backtesting and hyperopt will not work if this + method is used. + +*** ### Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. @@ -426,6 +475,7 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +*** ### Additional data (Trades) A history of Trades can be retrieved in the strategy by querying the database. From 6e86a47764d1d246d4603292c864547200fa5bbf Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Wed, 13 May 2020 14:49:16 +0400 Subject: [PATCH 7/7] updated docs --- docs/strategy-customization.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 20cde7556..d5bc76c65 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -383,13 +383,13 @@ if self.dp: ``` #### *current_whitelist()* -Imagine you've developed a strategy that trades the `1m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. +Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. The strategy might look something like this: -*Scan through the top 10 pairs by volume using the `VolumePairList` every minute and use a 14 day ATR to buy and sell.* +*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.* -Due to the limited available data, it's impossible to resample our `1m` candles into daily candles for use in the 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1/3 of a daily candle. We need 14 days at least! +Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. @@ -399,7 +399,7 @@ This is where calling `self.dp.current_whitelist()` comes in handy. class SampleStrategy(IStrategy): # strategy init stuff... - ticker_interval = '1m' + ticker_interval = '5m' # more strategy init stuff.. @@ -416,8 +416,7 @@ class SampleStrategy(IStrategy): informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') # Get the 14 day ATR. atr = ta.ATR(informative, timeperiod=14) - # Assign the Daily atr to the 1 minute dataframe. - dataframe['daily_atr'] = atr + # Do other stuff ``` #### *get_pair_dataframe(pair, timeframe)*