Split refresh from load_data/load_pair_history

freqtrade/data/history.py

@@ -128,39 +128,26 @@ def load_pair_history(pair: str,
                       timeframe: str,
                       datadir: Path,
                       timerange: Optional[TimeRange] = None,
-                      refresh_pairs: bool = False,
-                      exchange: Optional[Exchange] = None,
                       fill_up_missing: bool = True,
                       drop_incomplete: bool = True,
                       startup_candles: int = 0,
                       ) -> DataFrame:
     """
-    Loads cached ticker history for the given pair.
+    Load cached ticker history for the given pair.
+
     :param pair: Pair to load data for
     :param timeframe: Ticker timeframe (e.g. "5m")
     :param datadir: Path to the data storage location.
     :param timerange: Limit data to be loaded to this timerange
-    :param refresh_pairs: Refresh pairs from exchange.
-        (Note: Requires exchange to be passed as well.)
-    :param exchange: Exchange object (needed when using "refresh_pairs")
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param drop_incomplete: Drop last candle assuming it may be incomplete.
     :param startup_candles: Additional candles to load at the start of the period
     :return: DataFrame with ohlcv data, or empty DataFrame
     """
-
     timerange_startup = deepcopy(timerange)
     if startup_candles > 0 and timerange_startup:
         timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
 
-    # The user forced the refresh of pairs
-    if refresh_pairs:
-        _download_pair_history(datadir=datadir,
-                               exchange=exchange,
-                               pair=pair,
-                               timeframe=timeframe,
-                               timerange=timerange)
-
     pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup)
 
     if pairdata:
@@ -177,33 +164,53 @@ def load_pair_history(pair: str,
         return DataFrame()
 
 
+def refresh_pair_history(pair: str,
+                         timeframe: str,
+                         datadir: Path,
+                         exchange: Exchange,
+                         timerange: Optional[TimeRange] = None,
+                         startup_candles: int = 0,
+                         ) -> None:
+    """
+    Refresh cached ticker history for the given pair.
+
+    :param pair: Pair to load data for
+    :param timeframe: Ticker timeframe (e.g. "5m")
+    :param datadir: Path to the data storage location.
+    :param timerange: Limit data to be loaded to this timerange
+    :param exchange: Exchange object
+    :param startup_candles: Additional candles to load at the start of the period
+    """
+    timerange_startup = deepcopy(timerange)
+    if startup_candles > 0 and timerange_startup:
+        timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
+
+    _download_pair_history(datadir=datadir,
+                           exchange=exchange,
+                           pair=pair,
+                           timeframe=timeframe,
+                           timerange=timerange)
+
+
 def load_data(datadir: Path,
               timeframe: str,
               pairs: List[str],
-              refresh_pairs: bool = False,
-              exchange: Optional[Exchange] = None,
               timerange: Optional[TimeRange] = None,
               fill_up_missing: bool = True,
               startup_candles: int = 0,
               fail_without_data: bool = False
               ) -> Dict[str, DataFrame]:
     """
-    Loads ticker history data for a list of pairs
+    Load ticker history data for a list of pairs.
+
     :param datadir: Path to the data storage location.
     :param timeframe: Ticker Timeframe (e.g. "5m")
     :param pairs: List of pairs to load
-    :param refresh_pairs: Refresh pairs from exchange.
-        (Note: Requires exchange to be passed as well.)
-    :param exchange: Exchange object (needed when using "refresh_pairs")
     :param timerange: Limit data to be loaded to this timerange
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param startup_candles: Additional candles to load at the start of the period
     :param fail_without_data: Raise OperationalException if no data is found.
     :return: dict(<pair>:<Dataframe>)
-    TODO: refresh_pairs is still used by edge to keep the data uptodate.
-        This should be replaced in the future. Instead, writing the current candles to disk
-        from dataprovider should be implemented, as this would avoid loading ohlcv data twice.
-        exchange and refresh_pairs are then not needed here nor in load_pair_history.
     """
     result: Dict[str, DataFrame] = {}
     if startup_candles > 0 and timerange:
@@ -212,8 +219,6 @@ def load_data(datadir: Path,
     for pair in pairs:
         hist = load_pair_history(pair=pair, timeframe=timeframe,
                                  datadir=datadir, timerange=timerange,
-                                 refresh_pairs=refresh_pairs,
-                                 exchange=exchange,
                                  fill_up_missing=fill_up_missing,
                                  startup_candles=startup_candles)
         if not hist.empty:
@@ -224,6 +229,33 @@ def load_data(datadir: Path,
     return result
 
 
+def refresh_data(datadir: Path,
+                 timeframe: str,
+                 pairs: List[str],
+                 exchange: Exchange,
+                 timerange: Optional[TimeRange] = None,
+                 startup_candles: int = 0,
+                 ) -> None:
+    """
+    Refresh ticker history data for a list of pairs.
+
+    :param datadir: Path to the data storage location.
+    :param timeframe: Ticker Timeframe (e.g. "5m")
+    :param pairs: List of pairs to load
+    :param exchange: Exchange object
+    :param timerange: Limit data to be loaded to this timerange
+    :param startup_candles: Additional candles to load at the start of the period
+    """
+    if startup_candles > 0 and timerange:
+        logger.info(f'Using indicator startup period: {startup_candles} ...')
+
+    for pair in pairs:
+        refresh_pair_history(pair=pair, timeframe=timeframe,
+                             datadir=datadir, timerange=timerange,
+                             exchange=exchange,
+                             startup_candles=startup_candles)
+
+
 def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path:
     pair_s = pair.replace("/", "_")
     filename = datadir.joinpath(f'{pair_s}-{timeframe}.json')

freqtrade/edge/__init__.py

@@ -94,12 +94,20 @@ class Edge:
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using local backtesting data (using whitelist in given config) ...')
 
+        if self._refresh_pairs:
+            history.refresh_data(
+                datadir=Path(self.config['datadir']),
+                pairs=pairs,
+                exchange=self.exchange,
+                timeframe=self.strategy.ticker_interval,
+                timerange=self._timerange,
+                startup_candles=self.strategy.startup_candle_count,
+            )
+
         data = history.load_data(
             datadir=Path(self.config['datadir']),
             pairs=pairs,
             timeframe=self.strategy.ticker_interval,
-            refresh_pairs=self._refresh_pairs,
-            exchange=self.exchange,
             timerange=self._timerange,
             startup_candles=self.strategy.startup_candle_count,
         )

tests/data/test_history.py

@@ -21,6 +21,7 @@ from freqtrade.data.history import (_download_pair_history,
                                     pair_trades_filename,
                                     refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data,
+                                    refresh_data, refresh_pair_history,
                                     trim_dataframe, trim_tickerlist,
                                     validate_backtest_data)
 from freqtrade.exchange import timeframe_to_minutes
@@ -129,16 +130,17 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
     )
 
     # download a new pair if refresh_pairs is set
-    load_pair_history(datadir=testdatadir, timeframe='1m',
+    refresh_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC',
-                      refresh_pairs=True, exchange=exchange, pair='MEME/BTC')
+                         exchange=exchange)
+    load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC')
     assert file.is_file()
     assert log_has_re(
         'Download history data for pair: "MEME/BTC", timeframe: 1m '
         'and store in .*', caplog
     )
     with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'):
-        load_pair_history(datadir=testdatadir, timeframe='1m',
+        refresh_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC',
-                          refresh_pairs=True, exchange=None, pair='MEME/BTC')
+                             exchange=None)
     _clean_test_file(file)
 
 
@@ -372,12 +374,24 @@ def test_load_partial_missing(testdatadir, caplog) -> None:
 
 
 def test_init(default_conf, mocker) -> None:
-    exchange = get_patched_exchange(mocker, default_conf)
     assert {} == load_data(
         datadir='',
-        exchange=exchange,
         pairs=[],
-        refresh_pairs=True,
+        timeframe=default_conf['ticker_interval']
+    )
+
+
+def test_init_with_refresh(default_conf, mocker) -> None:
+    exchange = get_patched_exchange(mocker, default_conf)
+    refresh_data(
+        datadir='',
+        pairs=[],
+        timeframe=default_conf['ticker_interval'],
+        exchange=exchange
+    )
+    assert {} == load_data(
+        datadir='',
+        pairs=[],
         timeframe=default_conf['ticker_interval']
     )
 

tests/edge/test_edge.py

@@ -255,8 +255,8 @@ def test_edge_heartbeat_calculate(mocker, edge_conf):
     assert edge.calculate() is False
 
 
-def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False,
+def mocked_load_data(datadir, pairs=[], timeframe='0m',
-                     timerange=None, exchange=None, *args, **kwargs):
+                     timerange=None, *args, **kwargs):
     hz = 0.1
     base = 0.001
 
@@ -290,6 +290,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False,
 def test_edge_process_downloaded_data(mocker, edge_conf):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
+    mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
     mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
 
@@ -301,6 +302,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf):
 def test_edge_process_no_data(mocker, edge_conf, caplog):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
+    mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
     mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={}))
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
 
@@ -313,6 +315,7 @@ def test_edge_process_no_data(mocker, edge_conf, caplog):
 def test_edge_process_no_trades(mocker, edge_conf, caplog):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
+    mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
     mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
     # Return empty
     mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[]))

tests/optimize/test_backtesting.py

@@ -116,8 +116,8 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
     assert len(results) == num_results
 
 
-def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False,
+def mocked_load_data(datadir, pairs=[], timeframe='0m',
-                     timerange=None, exchange=None, live=False, *args, **kwargs):
+                     timerange=None, *args, **kwargs):
     tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC",
                                                        fill_missing=True)}