Merge pull request #2678 from hroff-1902/dataprovider-history-split-refresh

Dataprovider history: split refresh part

freqtrade/data/history.py

@@ -128,39 +128,26 @@ def load_pair_history(pair: str,
                       timeframe: str,
                       datadir: Path,
                       timerange: Optional[TimeRange] = None,
-                      refresh_pairs: bool = False,
-                      exchange: Optional[Exchange] = None,
                       fill_up_missing: bool = True,
                       drop_incomplete: bool = True,
                       startup_candles: int = 0,
                       ) -> DataFrame:
     """
-    Loads cached ticker history for the given pair.
+    Load cached ticker history for the given pair.
+
     :param pair: Pair to load data for
     :param timeframe: Ticker timeframe (e.g. "5m")
     :param datadir: Path to the data storage location.
     :param timerange: Limit data to be loaded to this timerange
-    :param refresh_pairs: Refresh pairs from exchange.
-        (Note: Requires exchange to be passed as well.)
-    :param exchange: Exchange object (needed when using "refresh_pairs")
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param drop_incomplete: Drop last candle assuming it may be incomplete.
     :param startup_candles: Additional candles to load at the start of the period
     :return: DataFrame with ohlcv data, or empty DataFrame
     """
-
     timerange_startup = deepcopy(timerange)
     if startup_candles > 0 and timerange_startup:
         timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
 
-    # The user forced the refresh of pairs
-    if refresh_pairs:
-        _download_pair_history(datadir=datadir,
-                               exchange=exchange,
-                               pair=pair,
-                               timeframe=timeframe,
-                               timerange=timerange)
-
     pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup)
 
     if pairdata:
@@ -180,30 +167,22 @@ def load_pair_history(pair: str,
 def load_data(datadir: Path,
               timeframe: str,
               pairs: List[str],
-              refresh_pairs: bool = False,
-              exchange: Optional[Exchange] = None,
               timerange: Optional[TimeRange] = None,
               fill_up_missing: bool = True,
               startup_candles: int = 0,
               fail_without_data: bool = False
               ) -> Dict[str, DataFrame]:
     """
-    Loads ticker history data for a list of pairs
+    Load ticker history data for a list of pairs.
+
     :param datadir: Path to the data storage location.
     :param timeframe: Ticker Timeframe (e.g. "5m")
     :param pairs: List of pairs to load
-    :param refresh_pairs: Refresh pairs from exchange.
-        (Note: Requires exchange to be passed as well.)
-    :param exchange: Exchange object (needed when using "refresh_pairs")
     :param timerange: Limit data to be loaded to this timerange
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param startup_candles: Additional candles to load at the start of the period
     :param fail_without_data: Raise OperationalException if no data is found.
     :return: dict(<pair>:<Dataframe>)
-    TODO: refresh_pairs is still used by edge to keep the data uptodate.
-        This should be replaced in the future. Instead, writing the current candles to disk
-        from dataprovider should be implemented, as this would avoid loading ohlcv data twice.
-        exchange and refresh_pairs are then not needed here nor in load_pair_history.
     """
     result: Dict[str, DataFrame] = {}
     if startup_candles > 0 and timerange:
@@ -212,8 +191,6 @@ def load_data(datadir: Path,
     for pair in pairs:
         hist = load_pair_history(pair=pair, timeframe=timeframe,
                                  datadir=datadir, timerange=timerange,
-                                 refresh_pairs=refresh_pairs,
-                                 exchange=exchange,
                                  fill_up_missing=fill_up_missing,
                                  startup_candles=startup_candles)
         if not hist.empty:
@@ -224,6 +201,27 @@ def load_data(datadir: Path,
     return result
 
 
+def refresh_data(datadir: Path,
+                 timeframe: str,
+                 pairs: List[str],
+                 exchange: Exchange,
+                 timerange: Optional[TimeRange] = None,
+                 ) -> None:
+    """
+    Refresh ticker history data for a list of pairs.
+
+    :param datadir: Path to the data storage location.
+    :param timeframe: Ticker Timeframe (e.g. "5m")
+    :param pairs: List of pairs to load
+    :param exchange: Exchange object
+    :param timerange: Limit data to be loaded to this timerange
+    """
+    for pair in pairs:
+        _download_pair_history(pair=pair, timeframe=timeframe,
+                               datadir=datadir, timerange=timerange,
+                               exchange=exchange)
+
+
 def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path:
     pair_s = pair.replace("/", "_")
     filename = datadir.joinpath(f'{pair_s}-{timeframe}.json')

freqtrade/edge/__init__.py

@@ -94,12 +94,19 @@ class Edge:
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using local backtesting data (using whitelist in given config) ...')
 
+        if self._refresh_pairs:
+            history.refresh_data(
+                datadir=Path(self.config['datadir']),
+                pairs=pairs,
+                exchange=self.exchange,
+                timeframe=self.strategy.ticker_interval,
+                timerange=self._timerange,
+            )
+
         data = history.load_data(
             datadir=Path(self.config['datadir']),
             pairs=pairs,
             timeframe=self.strategy.ticker_interval,
-            refresh_pairs=self._refresh_pairs,
-            exchange=self.exchange,
             timerange=self._timerange,
             startup_candles=self.strategy.startup_candle_count,
         )