diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 2c01f9f48..64f530b53 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -128,39 +128,26 @@ def load_pair_history(pair: str, timeframe: str, datadir: Path, timerange: Optional[TimeRange] = None, - refresh_pairs: bool = False, - exchange: Optional[Exchange] = None, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, ) -> DataFrame: """ - Loads cached ticker history for the given pair. + Load cached ticker history for the given pair. + :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param datadir: Path to the data storage location. :param timerange: Limit data to be loaded to this timerange - :param refresh_pairs: Refresh pairs from exchange. - (Note: Requires exchange to be passed as well.) - :param exchange: Exchange object (needed when using "refresh_pairs") :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :return: DataFrame with ohlcv data, or empty DataFrame """ - timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) - # The user forced the refresh of pairs - if refresh_pairs: - _download_pair_history(datadir=datadir, - exchange=exchange, - pair=pair, - timeframe=timeframe, - timerange=timerange) - pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) if pairdata: @@ -180,30 +167,22 @@ def load_pair_history(pair: str, def load_data(datadir: Path, timeframe: str, pairs: List[str], - refresh_pairs: bool = False, - exchange: Optional[Exchange] = None, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, startup_candles: int = 0, fail_without_data: bool = False ) -> Dict[str, DataFrame]: """ - Loads ticker history data for a list of pairs + Load ticker history data for a list of pairs. + :param datadir: Path to the data storage location. :param timeframe: Ticker Timeframe (e.g. "5m") :param pairs: List of pairs to load - :param refresh_pairs: Refresh pairs from exchange. - (Note: Requires exchange to be passed as well.) - :param exchange: Exchange object (needed when using "refresh_pairs") :param timerange: Limit data to be loaded to this timerange :param fill_up_missing: Fill missing values with "No action"-candles :param startup_candles: Additional candles to load at the start of the period :param fail_without_data: Raise OperationalException if no data is found. :return: dict(:) - TODO: refresh_pairs is still used by edge to keep the data uptodate. - This should be replaced in the future. Instead, writing the current candles to disk - from dataprovider should be implemented, as this would avoid loading ohlcv data twice. - exchange and refresh_pairs are then not needed here nor in load_pair_history. """ result: Dict[str, DataFrame] = {} if startup_candles > 0 and timerange: @@ -212,8 +191,6 @@ def load_data(datadir: Path, for pair in pairs: hist = load_pair_history(pair=pair, timeframe=timeframe, datadir=datadir, timerange=timerange, - refresh_pairs=refresh_pairs, - exchange=exchange, fill_up_missing=fill_up_missing, startup_candles=startup_candles) if not hist.empty: @@ -224,6 +201,27 @@ def load_data(datadir: Path, return result +def refresh_data(datadir: Path, + timeframe: str, + pairs: List[str], + exchange: Exchange, + timerange: Optional[TimeRange] = None, + ) -> None: + """ + Refresh ticker history data for a list of pairs. + + :param datadir: Path to the data storage location. + :param timeframe: Ticker Timeframe (e.g. "5m") + :param pairs: List of pairs to load + :param exchange: Exchange object + :param timerange: Limit data to be loaded to this timerange + """ + for pair in pairs: + _download_pair_history(pair=pair, timeframe=timeframe, + datadir=datadir, timerange=timerange, + exchange=exchange) + + def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: pair_s = pair.replace("/", "_") filename = datadir.joinpath(f'{pair_s}-{timeframe}.json') diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4bc3023a4..85029905b 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -94,12 +94,19 @@ class Edge: logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using local backtesting data (using whitelist in given config) ...') + if self._refresh_pairs: + history.refresh_data( + datadir=Path(self.config['datadir']), + pairs=pairs, + exchange=self.exchange, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + ) + data = history.load_data( datadir=Path(self.config['datadir']), pairs=pairs, timeframe=self.strategy.ticker_interval, - refresh_pairs=self._refresh_pairs, - exchange=self.exchange, timerange=self._timerange, startup_candles=self.strategy.startup_candle_count, ) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 708702ab2..c9b198b39 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -21,6 +21,7 @@ from freqtrade.data.history import (_download_pair_history, pair_trades_filename, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, + refresh_data, trim_dataframe, trim_tickerlist, validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes @@ -129,16 +130,17 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, ) # download a new pair if refresh_pairs is set - load_pair_history(datadir=testdatadir, timeframe='1m', - refresh_pairs=True, exchange=exchange, pair='MEME/BTC') + refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], + exchange=exchange) + load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): - load_pair_history(datadir=testdatadir, timeframe='1m', - refresh_pairs=True, exchange=None, pair='MEME/BTC') + refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], + exchange=None) _clean_test_file(file) @@ -372,12 +374,24 @@ def test_load_partial_missing(testdatadir, caplog) -> None: def test_init(default_conf, mocker) -> None: - exchange = get_patched_exchange(mocker, default_conf) assert {} == load_data( datadir='', - exchange=exchange, pairs=[], - refresh_pairs=True, + timeframe=default_conf['ticker_interval'] + ) + + +def test_init_with_refresh(default_conf, mocker) -> None: + exchange = get_patched_exchange(mocker, default_conf) + refresh_data( + datadir='', + pairs=[], + timeframe=default_conf['ticker_interval'], + exchange=exchange + ) + assert {} == load_data( + datadir='', + pairs=[], timeframe=default_conf['ticker_interval'] ) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index bdb986d6d..3a866c0a8 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -255,8 +255,8 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): assert edge.calculate() is False -def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False, - timerange=None, exchange=None, *args, **kwargs): +def mocked_load_data(datadir, pairs=[], timeframe='0m', + timerange=None, *args, **kwargs): hz = 0.1 base = 0.001 @@ -290,6 +290,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False, def test_edge_process_downloaded_data(mocker, edge_conf): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -301,6 +302,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): def test_edge_process_no_data(mocker, edge_conf, caplog): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -313,6 +315,7 @@ def test_edge_process_no_data(mocker, edge_conf, caplog): def test_edge_process_no_trades(mocker, edge_conf, caplog): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) # Return empty mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[])) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 5086891a6..38a95be7a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -116,8 +116,8 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: assert len(results) == num_results -def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False, - timerange=None, exchange=None, live=False, *args, **kwargs): +def mocked_load_data(datadir, pairs=[], timeframe='0m', + timerange=None, *args, **kwargs): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC", fill_missing=True)}