Merge pull request #1507 from xmatthias/feat/dataprovider

Data Provider

freqtrade/tests/data/test_dataprovider.py

@@ -0,0 +1,92 @@
+from unittest.mock import MagicMock
+
+from pandas import DataFrame
+
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.state import RunMode
+from freqtrade.tests.conftest import get_patched_exchange
+
+
+def test_ohlcv(mocker, default_conf, ticker_history):
+    default_conf["runmode"] = RunMode.DRY_RUN
+    tick_interval = default_conf["ticker_interval"]
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._klines[("XRP/BTC", tick_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.DRY_RUN
+    assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", tick_interval))
+    assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame)
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval) is not ticker_history
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval, copy=False) is ticker_history
+    assert not dp.ohlcv("UNITTEST/BTC", tick_interval).empty
+    assert dp.ohlcv("NONESENSE/AAA", tick_interval).empty
+
+    # Test with and without parameter
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval).equals(dp.ohlcv("UNITTEST/BTC"))
+
+    default_conf["runmode"] = RunMode.LIVE
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.LIVE
+    assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame)
+
+    default_conf["runmode"] = RunMode.BACKTEST
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.BACKTEST
+    assert dp.ohlcv("UNITTEST/BTC", tick_interval).empty
+
+
+def test_historic_ohlcv(mocker, default_conf, ticker_history):
+
+    historymock = MagicMock(return_value=ticker_history)
+    mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
+
+    # exchange = get_patched_exchange(mocker, default_conf)
+    dp = DataProvider(default_conf, None)
+    data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
+    assert isinstance(data, DataFrame)
+    assert historymock.call_count == 1
+    assert historymock.call_args_list[0][1]["datadir"] is None
+    assert historymock.call_args_list[0][1]["refresh_pairs"] is False
+    assert historymock.call_args_list[0][1]["ticker_interval"] == "5m"
+
+
+def test_available_pairs(mocker, default_conf, ticker_history):
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    tick_interval = default_conf["ticker_interval"]
+    exchange._klines[("XRP/BTC", tick_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history
+    dp = DataProvider(default_conf, exchange)
+
+    assert len(dp.available_pairs) == 2
+    assert dp.available_pairs == [
+        ("XRP/BTC", tick_interval),
+        ("UNITTEST/BTC", tick_interval),
+    ]
+
+
+def test_refresh(mocker, default_conf, ticker_history):
+    refresh_mock = MagicMock()
+    mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
+
+    exchange = get_patched_exchange(mocker, default_conf, id="binance")
+    tick_interval = default_conf["ticker_interval"]
+    pairs = [("XRP/BTC", tick_interval), ("UNITTEST/BTC", tick_interval)]
+
+    pairs_non_trad = [("ETH/USDT", tick_interval), ("BTC/TUSD", "1h")]
+
+    dp = DataProvider(default_conf, exchange)
+    dp.refresh(pairs)
+
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args[0]) == 1
+    assert len(refresh_mock.call_args[0][0]) == len(pairs)
+    assert refresh_mock.call_args[0][0] == pairs
+
+    refresh_mock.reset_mock()
+    dp.refresh(pairs, pairs_non_trad)
+    assert refresh_mock.call_count == 1
+    assert len(refresh_mock.call_args[0]) == 1
+    assert len(refresh_mock.call_args[0][0]) == len(pairs) + len(pairs_non_trad)
+    assert refresh_mock.call_args[0][0] == pairs + pairs_non_trad

freqtrade/tests/exchange/test_exchange.py

@@ -765,7 +765,7 @@ def test_get_history(default_conf, mocker, caplog):
     pair = 'ETH/BTC'
 
     async def mock_candle_hist(pair, tick_interval, since_ms):
-        return pair, tick
+        return pair, tick_interval, tick
 
     exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
     # one_call calculation * 1.8 should do 2 calls
@@ -778,7 +778,7 @@ def test_get_history(default_conf, mocker, caplog):
     assert len(ret) == 2
 
 
-def test_refresh_tickers(mocker, default_conf, caplog) -> None:
+def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
     tick = [
         [
             (arrow.utcnow().timestamp - 1) * 1000,  # unix timestamp ms
@@ -802,12 +802,12 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None:
     exchange = get_patched_exchange(mocker, default_conf)
     exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
 
-    pairs = ['IOTA/ETH', 'XRP/ETH']
+    pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]
     # empty dicts
     assert not exchange._klines
-    exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
+    exchange.refresh_latest_ohlcv(pairs)
 
-    assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
+    assert log_has(f'Refreshing ohlcv data for {len(pairs)} pairs', caplog.record_tuples)
     assert exchange._klines
     assert exchange._api_async.fetch_ohlcv.call_count == 2
     for pair in pairs:
@@ -822,10 +822,11 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None:
         assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False)
 
     # test caching
-    exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
+    exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')])
 
     assert exchange._api_async.fetch_ohlcv.call_count == 2
-    assert log_has(f"Using cached klines data for {pairs[0]} ...", caplog.record_tuples)
+    assert log_has(f"Using cached ohlcv data for {pairs[0][0]}, {pairs[0][1]} ...",
+                   caplog.record_tuples)
 
 
 @pytest.mark.asyncio
@@ -850,11 +851,12 @@ async def test__async_get_candle_history(default_conf, mocker, caplog):
     pair = 'ETH/BTC'
     res = await exchange._async_get_candle_history(pair, "5m")
     assert type(res) is tuple
-    assert len(res) == 2
+    assert len(res) == 3
     assert res[0] == pair
-    assert res[1] == tick
+    assert res[1] == "5m"
+    assert res[2] == tick
     assert exchange._api_async.fetch_ohlcv.call_count == 1
-    assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
+    assert not log_has(f"Using cached ohlcv data for {pair} ...", caplog.record_tuples)
 
     # exchange = Exchange(default_conf)
     await async_ccxt_exception(mocker, default_conf, MagicMock(),
@@ -883,48 +885,14 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
     pair = 'ETH/BTC'
     res = await exchange._async_get_candle_history(pair, "5m")
     assert type(res) is tuple
-    assert len(res) == 2
+    assert len(res) == 3
     assert res[0] == pair
-    assert res[1] == tick
+    assert res[1] == "5m"
+    assert res[2] == tick
     assert exchange._api_async.fetch_ohlcv.call_count == 1
 
 
-@pytest.mark.asyncio
-async def test_async_get_candles_history(default_conf, mocker):
-    tick = [
-        [
-            1511686200000,  # unix timestamp ms
-            1,  # open
-            2,  # high
-            3,  # low
-            4,  # close
-            5,  # volume (in quote currency)
-        ]
-    ]
-
-    async def mock_get_candle_hist(pair, tick_interval, since_ms=None):
-        return (pair, tick)
-
-    exchange = get_patched_exchange(mocker, default_conf)
-    # Monkey-patch async function
-    exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
-
-    exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist)
-
-    pairs = ['ETH/BTC', 'XRP/BTC']
-    res = await exchange.async_get_candles_history(pairs, "5m")
-    assert type(res) is list
-    assert len(res) == 2
-    assert type(res[0]) is tuple
-    assert res[0][0] == pairs[0]
-    assert res[0][1] == tick
-    assert res[1][0] == pairs[1]
-    assert res[1][1] == tick
-    assert exchange._async_get_candle_history.call_count == 2
-
-
-@pytest.mark.asyncio
-async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog):
+def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog):
 
     async def mock_get_candle_hist(pair, *args, **kwargs):
         if pair == 'ETH/BTC':
@@ -937,12 +905,16 @@ async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog
     # Monkey-patch async function with empty result
     exchange._api_async.fetch_ohlcv = MagicMock(side_effect=mock_get_candle_hist)
 
-    pairs = ['ETH/BTC', 'XRP/BTC']
-    res = await exchange.async_get_candles_history(pairs, "5m")
+    pairs = [("ETH/BTC", "5m"), ("XRP/BTC", "5m")]
+    res = exchange.refresh_latest_ohlcv(pairs)
+    assert exchange._klines
+    assert exchange._api_async.fetch_ohlcv.call_count == 2
+
     assert type(res) is list
     assert len(res) == 2
-    assert type(res[0]) is tuple
-    assert type(res[1]) is TypeError
+    # Test that each is in list at least once as order is not guaranteed
+    assert type(res[0]) is tuple or type(res[1]) is tuple
+    assert type(res[0]) is TypeError or type(res[1]) is TypeError
     assert log_has("Error loading ETH/BTC. Result was [[]].", caplog.record_tuples)
     assert log_has("Async code raised an exception: TypeError", caplog.record_tuples)
 
@@ -1010,7 +982,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker):
     # Test the ticker history sort
     res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval'])
     assert res[0] == 'ETH/BTC'
-    ticks = res[1]
+    ticks = res[2]
 
     assert sort_mock.call_count == 1
     assert ticks[0][0] == 1527830400000
@@ -1047,7 +1019,8 @@ async def test___async_get_candle_history_sort(default_conf, mocker):
     # Test the ticker history sort
     res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval'])
     assert res[0] == 'ETH/BTC'
-    ticks = res[1]
+    assert res[1] == default_conf['ticker_interval']
+    ticks = res[2]
     # Sorted not called again - data is already in order
     assert sort_mock.call_count == 0
     assert ticks[0][0] == 1527827700000

freqtrade/tests/optimize/test_backtesting.py

@@ -18,6 +18,7 @@ from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.optimize import get_timeframe
 from freqtrade.optimize.backtesting import (Backtesting, setup_configuration,
                                             start)
+from freqtrade.state import RunMode
 from freqtrade.strategy.default_strategy import DefaultStrategy
 from freqtrade.strategy.interface import SellType
 from freqtrade.tests.conftest import log_has, patch_exchange
@@ -200,6 +201,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
 
     assert 'timerange' not in config
     assert 'export' not in config
+    assert 'runmode' in config
+    assert config['runmode'] == RunMode.BACKTEST
 
 
 def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
@@ -230,6 +233,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
     assert 'exchange' in config
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
+    assert config['runmode'] == RunMode.BACKTEST
+
     assert log_has(
         'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples
@@ -445,7 +450,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
 
     mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
     mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe)
-    mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.optimize.backtesting.Backtesting',
@@ -480,7 +485,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None:
 
     mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={}))
     mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe)
-    mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.optimize.backtesting.Backtesting',

freqtrade/tests/optimize/test_edge_cli.py

@@ -7,6 +7,7 @@ from typing import List
 from freqtrade.edge import PairInfo
 from freqtrade.arguments import Arguments
 from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start)
+from freqtrade.state import RunMode
 from freqtrade.tests.conftest import log_has, patch_exchange
 
 
@@ -26,6 +27,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     ]
 
     config = setup_configuration(get_args(args))
+    assert config['runmode'] == RunMode.EDGECLI
+
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config
@@ -70,6 +73,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N
     assert 'exchange' in config
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
+    assert config['runmode'] == RunMode.EDGECLI
     assert log_has(
         'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples

freqtrade/tests/test_configuration.py

@@ -13,6 +13,7 @@ from freqtrade import OperationalException
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration, set_loggers
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
+from freqtrade.state import RunMode
 from freqtrade.tests.conftest import log_has
 
 
@@ -77,6 +78,8 @@ def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) ->
     assert validated_conf['max_open_trades'] > 999999999
     assert validated_conf['max_open_trades'] == float('inf')
     assert log_has('Validating configuration ...', caplog.record_tuples)
+    assert "runmode" in validated_conf
+    assert validated_conf['runmode'] == RunMode.DRY_RUN
 
 
 def test_load_config_file_exception(mocker) -> None:
@@ -177,6 +180,8 @@ def test_load_config_with_params(default_conf, mocker) -> None:
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
     assert validated_conf.get('db_url') == DEFAULT_DB_PROD_URL
+    assert "runmode" in validated_conf
+    assert validated_conf['runmode'] == RunMode.LIVE
 
     # Test args provided db_url dry_run
     conf = default_conf.copy()
@@ -365,8 +370,9 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non
 
     args = Arguments(arglist, '').get_parsed_arg()
 
-    configuration = Configuration(args)
+    configuration = Configuration(args, RunMode.BACKTEST)
     config = configuration.get_config()
+    assert config['runmode'] == RunMode.BACKTEST
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config
@@ -411,7 +417,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
     ]
     args = Arguments(arglist, '').get_parsed_arg()
 
-    configuration = Configuration(args)
+    configuration = Configuration(args, RunMode.HYPEROPT)
     config = configuration.get_config()
 
     assert 'epochs' in config
@@ -422,6 +428,8 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
     assert 'spaces' in config
     assert config['spaces'] == ['all']
     assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples)
+    assert "runmode" in config
+    assert config['runmode'] == RunMode.HYPEROPT
 
 
 def test_check_exchange(default_conf, caplog) -> None:

freqtrade/tests/test_freqtradebot.py

@@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     :return: None
     """
     freqtrade.strategy.get_signal = lambda e, s, t: value
-    freqtrade.exchange.refresh_tickers = lambda p, i: None
+    freqtrade.exchange.refresh_latest_ohlcv = lambda p: None
 
 
 def patch_RPCManager(mocker) -> MagicMock:
@@ -807,6 +807,37 @@ def test_process_trade_no_whitelist_pair(
     assert result is True
 
 
+def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+
+    def _refresh_whitelist(list):
+        return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC']
+
+    refresh_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_ticker=ticker,
+        get_markets=markets,
+        buy=MagicMock(side_effect=TemporaryError),
+        refresh_latest_ohlcv=refresh_mock,
+    )
+    inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m'), ("ETH/USDT", "1h")])
+    mocker.patch('time.sleep', return_value=None)
+
+    freqtrade = FreqtradeBot(default_conf)
+    freqtrade.pairlists._validate_whitelist = _refresh_whitelist
+    freqtrade.strategy.informative_pairs = inf_pairs
+    # patch_get_signal(freqtrade)
+
+    freqtrade._process()
+    assert inf_pairs.call_count == 1
+    assert refresh_mock.call_count == 1
+    assert ("BTC/ETH", "1m") in refresh_mock.call_args[0][0]
+    assert ("ETH/USDT", "1h") in refresh_mock.call_args[0][0]
+    assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0]
+
+
 def test_balance_fully_ask_side(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 0.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)