From 84a0f9ea42518d9e1dced4e027044365ec00a01c Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 17 Aug 2019 11:43:36 +0300 Subject: [PATCH 1/6] get_pair_dataframe helper method added --- freqtrade/data/dataprovider.py | 46 ++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b87589df7..e806f5aa7 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -44,36 +44,50 @@ class DataProvider(): def ohlcv(self, pair: str, ticker_interval: str = None, copy: bool = True) -> DataFrame: """ - get ohlcv data for the given pair as DataFrame - Please check `available_pairs` to verify which pairs are currently cached. + Get ohlcv data for the given pair as DataFrame + Please check `self.available_pairs` to verify which pairs are currently cached. :param pair: pair to get the data for - :param ticker_interval: ticker_interval to get pair for - :param copy: copy dataframe before returning. - Use false only for RO operations (where the dataframe is not modified) + :param ticker_interval: ticker interval to get data for + :param copy: copy dataframe before returning if True. + Use False only for read-only operations (where the dataframe is not modified) """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - if ticker_interval: - pairtick = (pair, ticker_interval) - else: - pairtick = (pair, self._config['ticker_interval']) + pairtick = (pair, ticker_interval or self._config['ticker_interval']) + if pairtick in self.available_pairs: + return self._exchange.klines(pairtick, copy=copy) - return self._exchange.klines(pairtick, copy=copy) - else: - return DataFrame() + return DataFrame() - def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: + def historic_ohlcv(self, pair: str, ticker_interval: str = None) -> DataFrame: """ - get stored historic ohlcv data + Get stored historic ohlcv data :param pair: pair to get the data for - :param ticker_interval: ticker_interval to get pair for + :param ticker_interval: ticker interval to get data for """ return load_pair_history(pair=pair, - ticker_interval=ticker_interval, + ticker_interval=ticker_interval or self._config['ticker_interval'], refresh_pairs=False, datadir=Path(self._config['datadir']) if self._config.get( 'datadir') else None ) + def get_pair_dataframe(self, pair: str, ticker_interval: str = None) -> DataFrame: + """ + Return pair ohlcv data, either live or cached historical -- depending + on the runmode. + :param pair: pair to get the data for + :param ticker_interval: ticker interval to get data for + """ + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + # Get live ohlcv data. + data = self.ohlcv(pair=pair, ticker_interval=ticker_interval) + else: + # Get historic ohlcv data (cached on disk). + data = self.historic_ohlcv(pair=pair, ticker_interval=ticker_interval) + if len(data) == 0: + logger.warning(f"No data found for pair {pair}") + return data + def ticker(self, pair: str): """ Return last ticker data From cda912bd8cc060b09168b68194b3a042b8f879ab Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 17 Aug 2019 12:59:27 +0300 Subject: [PATCH 2/6] test added --- freqtrade/tests/data/test_dataprovider.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 993f0b59b..54aab7052 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -51,6 +51,39 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): assert historymock.call_args_list[0][1]["ticker_interval"] == "5m" +def test_get_pair_dataframe(mocker, default_conf, ticker_history): + default_conf["runmode"] = RunMode.DRY_RUN + ticker_interval = default_conf["ticker_interval"] + exchange = get_patched_exchange(mocker, default_conf) + exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.DRY_RUN + assert ticker_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)) + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval) is not ticker_history + assert not dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval).empty + assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + + # Test with and without parameter + assert dp.get_pair_dataframe("UNITTEST/BTC", + ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC")) + + default_conf["runmode"] = RunMode.LIVE + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.LIVE + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) + assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + + historymock = MagicMock(return_value=ticker_history) + mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) + default_conf["runmode"] = RunMode.BACKTEST + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.BACKTEST + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) + # assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + + def test_available_pairs(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) From 8a2a8ab8b5da8392857898684cf8d983e3cd1881 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 18 Aug 2019 12:47:19 +0300 Subject: [PATCH 3/6] docstring for ohlcv improved --- freqtrade/data/dataprovider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index e806f5aa7..b67aba045 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -45,7 +45,7 @@ class DataProvider(): def ohlcv(self, pair: str, ticker_interval: str = None, copy: bool = True) -> DataFrame: """ Get ohlcv data for the given pair as DataFrame - Please check `self.available_pairs` to verify which pairs are currently cached. + Please use the `available_pairs` method to verify which pairs are currently cached. :param pair: pair to get the data for :param ticker_interval: ticker interval to get data for :param copy: copy dataframe before returning if True. From 310e4387065bf1407c8226a0bec9f79dc62e98ec Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 18 Aug 2019 12:55:31 +0300 Subject: [PATCH 4/6] logging message improved --- freqtrade/data/dataprovider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b67aba045..b904ba985 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -85,7 +85,7 @@ class DataProvider(): # Get historic ohlcv data (cached on disk). data = self.historic_ohlcv(pair=pair, ticker_interval=ticker_interval) if len(data) == 0: - logger.warning(f"No data found for pair {pair}") + logger.warning(f"No data found for ({pair}, {ticker_interval}).") return data def ticker(self, pair: str): From 407a3bca6222bd820d55656e73398dc4e6bef102 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 18 Aug 2019 13:00:37 +0300 Subject: [PATCH 5/6] implementation of ohlcv optimized --- freqtrade/data/dataprovider.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b904ba985..5b71c21a8 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -52,11 +52,10 @@ class DataProvider(): Use False only for read-only operations (where the dataframe is not modified) """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - pairtick = (pair, ticker_interval or self._config['ticker_interval']) - if pairtick in self.available_pairs: - return self._exchange.klines(pairtick, copy=copy) - - return DataFrame() + return self._exchange.klines((pair, ticker_interval or self._config['ticker_interval']), + copy=copy) + else: + return DataFrame() def historic_ohlcv(self, pair: str, ticker_interval: str = None) -> DataFrame: """ From d300964691977d34bc3618a8e75bb48b6ce863ae Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 18 Aug 2019 13:06:21 +0300 Subject: [PATCH 6/6] code formatting in test_dataprovider.py --- freqtrade/tests/data/test_dataprovider.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 54aab7052..2272f69a3 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -13,6 +13,7 @@ def test_ohlcv(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history + dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", ticker_interval)) @@ -37,11 +38,9 @@ def test_ohlcv(mocker, default_conf, ticker_history): def test_historic_ohlcv(mocker, default_conf, ticker_history): - historymock = MagicMock(return_value=ticker_history) mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) - # exchange = get_patched_exchange(mocker, default_conf) dp = DataProvider(default_conf, None) data = dp.historic_ohlcv("UNITTEST/BTC", "5m") assert isinstance(data, DataFrame) @@ -57,6 +56,7 @@ def test_get_pair_dataframe(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history + dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ticker_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)) @@ -86,12 +86,11 @@ def test_get_pair_dataframe(mocker, default_conf, ticker_history): def test_available_pairs(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) - ticker_interval = default_conf["ticker_interval"] exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history - dp = DataProvider(default_conf, exchange) + dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 assert dp.available_pairs == [ ("XRP/BTC", ticker_interval),