Merge pull request #1456 from freqtrade/fix/refresh_pairs_cached

Fix bug on --refresh-pairs-cached

freqtrade/configuration.py

@@ -152,13 +152,16 @@ class Configuration(object):
 
         return config
 
-    def _create_default_datadir(self, config: Dict[str, Any]) -> str:
+    def _create_datadir(self, config: Dict[str, Any], datadir: Optional[str] = None) -> str:
-        exchange_name = config.get('exchange', {}).get('name').lower()
+        if not datadir:
-        default_path = os.path.join('user_data', 'data', exchange_name)
+            # set datadir
-        if not os.path.isdir(default_path):
+            exchange_name = config.get('exchange', {}).get('name').lower()
-            os.makedirs(default_path)
+            datadir = os.path.join('user_data', 'data', exchange_name)
-            logger.info(f'Created data directory: {default_path}')
+
-        return default_path
+        if not os.path.isdir(datadir):
+            os.makedirs(datadir)
+            logger.info(f'Created data directory: {datadir}')
+        return datadir
 
     def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
         """
@@ -198,9 +201,9 @@ class Configuration(object):
 
         # If --datadir is used we add it to the configuration
         if 'datadir' in self.args and self.args.datadir:
-            config.update({'datadir': self.args.datadir})
+            config.update({'datadir': self._create_datadir(config, self.args.datadir)})
         else:
-            config.update({'datadir': self._create_default_datadir(config)})
+            config.update({'datadir': self._create_datadir(config, None)})
         logger.info('Using data folder: %s ...', config.get('datadir'))
 
         # If -r/--refresh-pairs-cached is used we add it to the configuration

freqtrade/data/history.py

@@ -88,20 +88,21 @@ def load_pair_history(pair: str,
     :return: DataFrame with ohlcv data
     """
 
-    pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
     # If the user force the refresh of pairs
     if refresh_pairs:
         if not exchange:
             raise OperationalException("Exchange needs to be initialized when "
                                        "calling load_data with refresh_pairs=True")
 
-        logger.info('Download data for all pairs and store them in %s', datadir)
+        logger.info('Download data for pair and store them in %s', datadir)
         download_pair_history(datadir=datadir,
                               exchange=exchange,
                               pair=pair,
                               tick_interval=ticker_interval,
                               timerange=timerange)
 
+    pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
+
     if pairdata:
         if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000:
             logger.warning('Missing data at start for pair %s, data starts at %s',

freqtrade/tests/optimize/test_backtesting.py

@@ -202,10 +202,11 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     assert 'export' not in config
 
 
-def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None:
+def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
     mocker.patch('freqtrade.configuration.open', mocker.mock_open(
         read_data=json.dumps(default_conf)
     ))
+    mocker.patch('freqtrade.configuration.Configuration._create_datadir', lambda s, c, x: x)
 
     args = [
         '--config', 'config.json',

freqtrade/tests/optimize/test_edge_cli.py

@@ -46,10 +46,11 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     assert 'stoploss_range' not in config
 
 
-def test_setup_configuration_with_arguments(mocker, edge_conf, caplog) -> None:
+def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> None:
     mocker.patch('freqtrade.configuration.open', mocker.mock_open(
         read_data=json.dumps(edge_conf)
     ))
+    mocker.patch('freqtrade.configuration.Configuration._create_datadir', lambda s, c, x: x)
 
     args = [
         '--config', 'config.json',

freqtrade/tests/test_configuration.py

@@ -247,6 +247,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
     mocker.patch('freqtrade.configuration.open', mocker.mock_open(
         read_data=json.dumps(default_conf)
     ))
+    mocker.patch('freqtrade.configuration.Configuration._create_datadir', lambda s, c, x: x)
 
     arglist = [
         '--config', 'config.json',
@@ -487,3 +488,13 @@ def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None:
 
 def test_validate_default_conf(default_conf) -> None:
     validate(default_conf, constants.CONF_SCHEMA, Draft4Validator)
+
+
+def test__create_datadir(mocker, default_conf, caplog) -> None:
+    mocker.patch('os.path.isdir', MagicMock(return_value=False))
+    md = MagicMock()
+    mocker.patch('os.makedirs', md)
+    cfg = Configuration(Namespace())
+    cfg._create_datadir(default_conf, '/foo/bar')
+    assert md.call_args[0][0] == "/foo/bar"
+    assert log_has('Created data directory: /foo/bar', caplog.record_tuples)