From a8f523adae66fb0eb4f28ffff4cd8394c7b6b0d4 Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Mon, 11 May 2020 19:32:28 +0400 Subject: [PATCH 01/11] attached pairlist manager onto dataprovider init for unified access to dynamic whitelist --- freqtrade/data/dataprovider.py | 12 +++++---- freqtrade/freqtradebot.py | 6 ++--- tests/data/test_dataprovider.py | 46 ++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index af0914939..01397d6b7 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode @@ -18,9 +19,10 @@ logger = logging.getLogger(__name__) class DataProvider: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange + self._pairlists: Optional = pairlists def refresh(self, pairlist: List[Tuple[str, str]], @@ -125,8 +127,8 @@ class DataProvider: As available pairs does not show whitelist until after informative pairs have been cached. :return: list of pairs in whitelist """ - from freqtrade.pairlist.pairlistmanager import PairListManager - pairlists = PairListManager(self._exchange, self._config) - pairlists.refresh_pairlist() - return pairlists.whitelist + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException("Dataprovider was not initialized with a pairlist provider.") diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4f4b3e3bb..73f0873e4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -71,15 +71,15 @@ class FreqtradeBot: self.wallets = Wallets(self.config, self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + self.pairlists = PairListManager(self.exchange, self.config) + + self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - self.pairlists = PairListManager(self.exchange, self.config) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index e5af80bc8..247703619 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,10 +1,12 @@ -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock, patch from pandas import DataFrame +import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from tests.conftest import get_patched_exchange, get_patched_freqtradebot +from tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ohlcv_history): @@ -151,21 +153,29 @@ def test_market(mocker, default_conf, markets): assert res is None -def test_current_whitelist(mocker, shitcoinmarkets, tickers, default_conf): - default_conf.update( - {"pairlists": [{"method": "VolumePairList", - "number_assets": 10, - "sort_key": "quoteVolume"}], }, ) - default_conf['exchange']['pair_blacklist'] = ['BLK/BTC'] +@patch('freqtrade.pairlist.pairlistmanager.PairListManager') +@patch('freqtrade.exchange.Exchange') +def test_current_whitelist(exchange, PairListManager, default_conf): + # patch default conf to volumepairlist + default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} - mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, - exchange_has=MagicMock(return_value=True), ) - bot = get_patched_freqtradebot(mocker, default_conf) - # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=shitcoinmarkets), ) - # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC', 'FUEL/BTC'] + pairlist = PairListManager(exchange, default_conf) + dp = DataProvider(default_conf, exchange, pairlist) - current_wl = bot.dataprovider.current_whitelist() - assert whitelist == current_wl + # Simulate volumepairs from exchange. + # pairlist.refresh_pairlist() + # Set the pairs manually... this would be done in refresh pairlist default whitelist + volumePL - blacklist + default_whitelist = default_conf['exchange']['pair_whitelist'] + default_blacklist = default_conf['exchange']['pair_blacklist'] + volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] + current_whitelist = list(set(volume_pairlist + default_whitelist)) + for pair in default_blacklist: + if pair in current_whitelist: + current_whitelist.remove(pair) + pairlist._whitelist = current_whitelist + + assert dp.current_whitelist() == pairlist._whitelist + + with pytest.raises(OperationalException) as e: + dp = DataProvider(default_conf, exchange) + dp.current_whitelist() From bc9efc31ad6c6e0806111384e6fa8229285d298a Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Wed, 6 May 2020 19:48:57 +0400 Subject: [PATCH 02/11] Added Method for accessing current pair list on initialization for dynamic informative pairs moved import into function to avoid circular import with hyperopt --- freqtrade/data/dataprovider.py | 14 ++++++++++++++ tests/data/test_dataprovider.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 1df710152..af0914939 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -116,3 +116,17 @@ class DataProvider: can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". """ return RunMode(self._config.get('runmode', RunMode.OTHER)) + + def current_whitelist(self) -> List[str]: + """ + fetch latest available whitelist. + + Useful when you have a large whitelist and need to call each pair as an informative pair. + As available pairs does not show whitelist until after informative pairs have been cached. + :return: list of pairs in whitelist + """ + from freqtrade.pairlist.pairlistmanager import PairListManager + + pairlists = PairListManager(self._exchange, self._config) + pairlists.refresh_pairlist() + return pairlists.whitelist diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 2b3dda188..e5af80bc8 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,10 +1,10 @@ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider from freqtrade.state import RunMode -from tests.conftest import get_patched_exchange +from tests.conftest import get_patched_exchange, get_patched_freqtradebot def test_ohlcv(mocker, default_conf, ohlcv_history): @@ -64,8 +64,8 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history): assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty # Test with and without parameter - assert dp.get_pair_dataframe("UNITTEST/BTC", - ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC")) + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\ + .equals(dp.get_pair_dataframe("UNITTEST/BTC")) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) @@ -90,10 +90,7 @@ def test_available_pairs(mocker, default_conf, ohlcv_history): dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == [ - ("XRP/BTC", ticker_interval), - ("UNITTEST/BTC", ticker_interval), - ] + assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ] def test_refresh(mocker, default_conf, ohlcv_history): @@ -152,3 +149,23 @@ def test_market(mocker, default_conf, markets): res = dp.market('UNITTEST/BTC') assert res is None + + +def test_current_whitelist(mocker, shitcoinmarkets, tickers, default_conf): + default_conf.update( + {"pairlists": [{"method": "VolumePairList", + "number_assets": 10, + "sort_key": "quoteVolume"}], }, ) + default_conf['exchange']['pair_blacklist'] = ['BLK/BTC'] + + mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, + exchange_has=MagicMock(return_value=True), ) + bot = get_patched_freqtradebot(mocker, default_conf) + # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=shitcoinmarkets), ) + # argument: use the whitelist dynamically by exchange-volume + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC', 'FUEL/BTC'] + + current_wl = bot.dataprovider.current_whitelist() + assert whitelist == current_wl From a5bfa5515cbdbab500dbe9df614730107804bacd Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Mon, 11 May 2020 20:13:06 +0400 Subject: [PATCH 03/11] Fix flake8 mypy --- freqtrade/data/dataprovider.py | 2 +- tests/data/test_dataprovider.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 01397d6b7..984652e24 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -22,7 +22,7 @@ class DataProvider: def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange - self._pairlists: Optional = pairlists + self._pairlists = pairlists def refresh(self, pairlist: List[Tuple[str, str]], diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 247703619..45ce1c009 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -164,7 +164,9 @@ def test_current_whitelist(exchange, PairListManager, default_conf): # Simulate volumepairs from exchange. # pairlist.refresh_pairlist() - # Set the pairs manually... this would be done in refresh pairlist default whitelist + volumePL - blacklist + + # Set the pairs manually... this would be done in refresh pairlist + # default whitelist + volumePL - blacklist default_whitelist = default_conf['exchange']['pair_whitelist'] default_blacklist = default_conf['exchange']['pair_blacklist'] volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] @@ -176,6 +178,6 @@ def test_current_whitelist(exchange, PairListManager, default_conf): assert dp.current_whitelist() == pairlist._whitelist - with pytest.raises(OperationalException) as e: + with pytest.raises(OperationalException): dp = DataProvider(default_conf, exchange) dp.current_whitelist() From 9fbe1357902aa90243309781f164e7328fdc5b9a Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Mon, 11 May 2020 19:32:28 +0400 Subject: [PATCH 04/11] attached pairlist manager onto dataprovider init for unified access to dynamic whitelist --- freqtrade/data/dataprovider.py | 12 +++++---- freqtrade/freqtradebot.py | 6 ++--- tests/data/test_dataprovider.py | 48 ++++++++++++++++++++------------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index af0914939..984652e24 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode @@ -18,9 +19,10 @@ logger = logging.getLogger(__name__) class DataProvider: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange + self._pairlists = pairlists def refresh(self, pairlist: List[Tuple[str, str]], @@ -125,8 +127,8 @@ class DataProvider: As available pairs does not show whitelist until after informative pairs have been cached. :return: list of pairs in whitelist """ - from freqtrade.pairlist.pairlistmanager import PairListManager - pairlists = PairListManager(self._exchange, self._config) - pairlists.refresh_pairlist() - return pairlists.whitelist + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException("Dataprovider was not initialized with a pairlist provider.") diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4f4b3e3bb..73f0873e4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -71,15 +71,15 @@ class FreqtradeBot: self.wallets = Wallets(self.config, self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + self.pairlists = PairListManager(self.exchange, self.config) + + self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - self.pairlists = PairListManager(self.exchange, self.config) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index e5af80bc8..45ce1c009 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,10 +1,12 @@ -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock, patch from pandas import DataFrame +import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from tests.conftest import get_patched_exchange, get_patched_freqtradebot +from tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ohlcv_history): @@ -151,21 +153,31 @@ def test_market(mocker, default_conf, markets): assert res is None -def test_current_whitelist(mocker, shitcoinmarkets, tickers, default_conf): - default_conf.update( - {"pairlists": [{"method": "VolumePairList", - "number_assets": 10, - "sort_key": "quoteVolume"}], }, ) - default_conf['exchange']['pair_blacklist'] = ['BLK/BTC'] +@patch('freqtrade.pairlist.pairlistmanager.PairListManager') +@patch('freqtrade.exchange.Exchange') +def test_current_whitelist(exchange, PairListManager, default_conf): + # patch default conf to volumepairlist + default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} - mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, - exchange_has=MagicMock(return_value=True), ) - bot = get_patched_freqtradebot(mocker, default_conf) - # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=shitcoinmarkets), ) - # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC', 'FUEL/BTC'] + pairlist = PairListManager(exchange, default_conf) + dp = DataProvider(default_conf, exchange, pairlist) - current_wl = bot.dataprovider.current_whitelist() - assert whitelist == current_wl + # Simulate volumepairs from exchange. + # pairlist.refresh_pairlist() + + # Set the pairs manually... this would be done in refresh pairlist + # default whitelist + volumePL - blacklist + default_whitelist = default_conf['exchange']['pair_whitelist'] + default_blacklist = default_conf['exchange']['pair_blacklist'] + volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] + current_whitelist = list(set(volume_pairlist + default_whitelist)) + for pair in default_blacklist: + if pair in current_whitelist: + current_whitelist.remove(pair) + pairlist._whitelist = current_whitelist + + assert dp.current_whitelist() == pairlist._whitelist + + with pytest.raises(OperationalException): + dp = DataProvider(default_conf, exchange) + dp.current_whitelist() From e864db1843c378b0eb8c6936f1b8a4aa3230eddc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 May 2020 06:38:14 +0200 Subject: [PATCH 05/11] Update test for dp.current_whitelist --- tests/data/test_dataprovider.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 45ce1c009..3e42abb95 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,9 +1,10 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from pandas import DataFrame import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from tests.conftest import get_patched_exchange @@ -153,30 +154,24 @@ def test_market(mocker, default_conf, markets): assert res is None -@patch('freqtrade.pairlist.pairlistmanager.PairListManager') -@patch('freqtrade.exchange.Exchange') -def test_current_whitelist(exchange, PairListManager, default_conf): +def test_current_whitelist(mocker, default_conf, tickers): # patch default conf to volumepairlist default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + get_tickers=tickers) + exchange = get_patched_exchange(mocker, default_conf) + pairlist = PairListManager(exchange, default_conf) dp = DataProvider(default_conf, exchange, pairlist) # Simulate volumepairs from exchange. - # pairlist.refresh_pairlist() - - # Set the pairs manually... this would be done in refresh pairlist - # default whitelist + volumePL - blacklist - default_whitelist = default_conf['exchange']['pair_whitelist'] - default_blacklist = default_conf['exchange']['pair_blacklist'] - volume_pairlist = ['ETH/BTC', 'LINK/BTC', 'ZRX/BTC', 'BCH/BTC', 'XRP/BTC'] - current_whitelist = list(set(volume_pairlist + default_whitelist)) - for pair in default_blacklist: - if pair in current_whitelist: - current_whitelist.remove(pair) - pairlist._whitelist = current_whitelist + pairlist.refresh_pairlist() assert dp.current_whitelist() == pairlist._whitelist + # The identity of the 2 lists should be identical + assert dp.current_whitelist() is pairlist._whitelist with pytest.raises(OperationalException): dp = DataProvider(default_conf, exchange) From 63dfe3669ff994b535adde5f1609bbba283de69f Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Wed, 13 May 2020 00:25:57 +0400 Subject: [PATCH 06/11] Updated docs for #3267 --- docs/strategy-customization.md | 160 +++++++++++++++++++++------------ 1 file changed, 105 insertions(+), 55 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index c4fc55811..20cde7556 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -324,62 +324,9 @@ class Awesomestrategy(IStrategy): !!! Note If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. -### Additional data (DataProvider) +*** -The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. - -All methods return `None` in case of failure (do not raise an exception). - -Please always check the mode of operation to select the correct method to get data (samples see below). - -#### Possible options for DataProvider - -- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval). -- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. -- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. -- `get_pair_dataframe(pair, timeframe)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). -- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. -- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. -- `runmode` - Property containing the current runmode. - -#### Example: fetch live / historical candle (OHLCV) data for the first informative pair - -``` python -if self.dp: - inf_pair, inf_timeframe = self.informative_pairs()[0] - informative = self.dp.get_pair_dataframe(pair=inf_pair, - timeframe=inf_timeframe) -``` - -!!! Warning "Warning about backtesting" - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` - for the backtesting runmode) provides the full time-range in one go, - so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). - -!!! Warning "Warning in hyperopt" - This option cannot currently be used during hyperopt. - -#### Orderbook - -``` python -if self.dp: - if self.dp.runmode.value in ('live', 'dry_run'): - ob = self.dp.orderbook(metadata['pair'], 1) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] -``` - -!!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work if this - method is used. - -#### Available Pairs - -``` python -if self.dp: - for pair, timeframe in self.dp.available_pairs: - print(f"available {pair}, {timeframe}") -``` +### Additional data (informative_pairs) #### Get data for non-tradeable pairs @@ -404,6 +351,108 @@ def informative_pairs(self): It is however better to use resampling to longer time-intervals when possible to avoid hammering the exchange with too many requests and risk being blocked. +*** + +### Additional data (DataProvider) + +The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. + +All methods return `None` in case of failure (do not raise an exception). + +Please always check the mode of operation to select the correct method to get data (samples see below). + +#### Possible options for DataProvider + +- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval). +- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist) +- [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). +- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. +- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. +- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. +- [`orderbook(pair, maximum)`](#orderbookpair-maximum) - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. +- `runmode` - Property containing the current runmode. + +#### Example Usages: + +#### *available_pairs* + +``` python +if self.dp: + for pair, timeframe in self.dp.available_pairs: + print(f"available {pair}, {timeframe}") +``` + +#### *current_whitelist()* +Imagine you've developed a strategy that trades the `1m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. + +The strategy might look something like this: + +*Scan through the top 10 pairs by volume using the `VolumePairList` every minute and use a 14 day ATR to buy and sell.* + +Due to the limited available data, it's impossible to resample our `1m` candles into daily candles for use in the 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1/3 of a daily candle. We need 14 days at least! + +Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. + +This is where calling `self.dp.current_whitelist()` comes in handy. + +```python +class SampleStrategy(IStrategy): + # strategy init stuff... + + ticker_interval = '1m' + + # more strategy init stuff.. + + def informative_pairs(self): + + # get access to all pairs available in whitelist. + pairs = self.dp.current_whitelist() + # Assign tf to each pair so they can be downloaded and cached for strategy. + informative_pairs = [(pair, '1d') for pair in pairs] + return informative_pairs + + def populate_indicators(self, dataframe, metadata): + # Get the informative pair + informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') + # Get the 14 day ATR. + atr = ta.ATR(informative, timeperiod=14) + # Assign the Daily atr to the 1 minute dataframe. + dataframe['daily_atr'] = atr +``` + +#### *get_pair_dataframe(pair, timeframe)* + +``` python +# fetch live / historical candle (OHLCV) data for the first informative pair +if self.dp: + inf_pair, inf_timeframe = self.informative_pairs()[0] + informative = self.dp.get_pair_dataframe(pair=inf_pair, + timeframe=inf_timeframe) +``` + +!!! Warning "Warning about backtesting" + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + for the backtesting runmode) provides the full time-range in one go, + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + +!!! Warning "Warning in hyperopt" + This option cannot currently be used during hyperopt. + +#### *orderbook(pair, maximum)* + +``` python +if self.dp: + if self.dp.runmode.value in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] +``` + +!!! Warning + The order book is not part of the historic data which means backtesting and hyperopt will not work if this + method is used. + +*** ### Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. @@ -426,6 +475,7 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +*** ### Additional data (Trades) A history of Trades can be retrieved in the strategy by querying the database. From 6e86a47764d1d246d4603292c864547200fa5bbf Mon Sep 17 00:00:00 2001 From: "Paul D. Mendes" Date: Wed, 13 May 2020 14:49:16 +0400 Subject: [PATCH 07/11] updated docs --- docs/strategy-customization.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 20cde7556..d5bc76c65 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -383,13 +383,13 @@ if self.dp: ``` #### *current_whitelist()* -Imagine you've developed a strategy that trades the `1m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. +Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. The strategy might look something like this: -*Scan through the top 10 pairs by volume using the `VolumePairList` every minute and use a 14 day ATR to buy and sell.* +*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.* -Due to the limited available data, it's impossible to resample our `1m` candles into daily candles for use in the 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1/3 of a daily candle. We need 14 days at least! +Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. @@ -399,7 +399,7 @@ This is where calling `self.dp.current_whitelist()` comes in handy. class SampleStrategy(IStrategy): # strategy init stuff... - ticker_interval = '1m' + ticker_interval = '5m' # more strategy init stuff.. @@ -416,8 +416,7 @@ class SampleStrategy(IStrategy): informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') # Get the 14 day ATR. atr = ta.ATR(informative, timeperiod=14) - # Assign the Daily atr to the 1 minute dataframe. - dataframe['daily_atr'] = atr + # Do other stuff ``` #### *get_pair_dataframe(pair, timeframe)* From 6ff457e3911fecb1e471b1b4950c277eefc26300 Mon Sep 17 00:00:00 2001 From: Pedro Torres Date: Wed, 13 May 2020 20:50:11 +0800 Subject: [PATCH 08/11] Update doc Ta_lib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the document with the latest Ta_lib-0.4.18. 👍 The requirements-common.txt has been updated previously on commit a379e68cf441407c9c04264714bfc7dbcc8e635d --- docs/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 88e2ef6eb..f017bef96 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -248,14 +248,14 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl` (make sure to use the version matching your python version) +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) ```cmd >cd \path\freqtrade-develop >python -m venv .env >.env\Scripts\activate.bat REM optionally install ta-lib from wheel -REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl +REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl >pip install -r requirements.txt >pip install -e . >freqtrade From b3dd0a68d56f179eb11a6d3996420a1490922836 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 13 May 2020 18:51:38 +0300 Subject: [PATCH 09/11] minor: fix typo in configuration.md --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 67e8578dd..eb7f02d5c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,7 +108,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
**Datatype:** Boolean | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName | `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String -| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Intege +| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Integer | `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
**Datatype:** Positive Integer or 0 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
**Datatype:** Boolean | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String From ced812660bf6a0aeb41923ca82fada4c5df5c52c Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 13 May 2020 22:37:50 +0300 Subject: [PATCH 10/11] Minor fix in the strategy docs --- docs/strategy-customization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index d5bc76c65..dd451128c 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -331,7 +331,7 @@ class Awesomestrategy(IStrategy): #### Get data for non-tradeable pairs Data for additional, informative pairs (reference pairs) can be beneficial for some strategies. -Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above). +Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below). These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting. The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument. From b974e8fddf86e1a10f63e05986201d29a16f8299 Mon Sep 17 00:00:00 2001 From: Sourcery AI Date: Thu, 14 May 2020 04:47:11 +0000 Subject: [PATCH 11/11] Refactored by Sourcery --- freqtrade/freqtradebot.py | 49 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 73f0873e4..eda73a8c2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -946,32 +946,31 @@ class FreqtradeBot: :return: Reason for cancel """ # if trade is not partially completed, just cancel the trade - if order['remaining'] == order['amount'] or order.get('filled') == 0.0: - if not self.exchange.check_order_canceled_empty(order): - reason = "cancelled due to timeout" - try: - # if trade is not partially completed, just delete the trade - self.exchange.cancel_order(trade.open_order_id, trade.pair) - except InvalidOrderException: - logger.exception(f"Could not cancel sell order {trade.open_order_id}") - return 'error cancelling order' - logger.info('Sell order %s for %s.', reason, trade) - else: - reason = "cancelled on exchange" - logger.info('Sell order %s for %s.', reason, trade) + if not ( + order['remaining'] == order['amount'] or order.get('filled') == 0.0 + ): + # TODO: figure out how to handle partially complete sell orders + return 'partially filled - keeping order open' + if self.exchange.check_order_canceled_empty(order): + reason = "cancelled on exchange" + else: + reason = "cancelled due to timeout" + try: + # if trade is not partially completed, just delete the trade + self.exchange.cancel_order(trade.open_order_id, trade.pair) + except InvalidOrderException: + logger.exception(f"Could not cancel sell order {trade.open_order_id}") + return 'error cancelling order' + logger.info('Sell order %s for %s.', reason, trade) + trade.close_rate = None + trade.close_rate_requested = None + trade.close_profit = None + trade.close_profit_abs = None + trade.close_date = None + trade.is_open = True + trade.open_order_id = None - trade.close_rate = None - trade.close_rate_requested = None - trade.close_profit = None - trade.close_profit_abs = None - trade.close_date = None - trade.is_open = True - trade.open_order_id = None - - return reason - - # TODO: figure out how to handle partially complete sell orders - return 'partially filled - keeping order open' + return reason def _safe_sell_amount(self, pair: str, amount: float) -> float: """