Document hdf5 dataformat

freqtrade/data/history/hdf5datahandler.py

@@ -59,6 +59,7 @@ class HDF5DataHandler(IDataHandler):
         _data = data.copy()
 
         filename = self._pair_data_filename(self._datadir, pair, timeframe)
+
         ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc')
         ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date'])
 
@@ -139,6 +140,7 @@ class HDF5DataHandler(IDataHandler):
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         key = self._pair_trades_key(pair)
+
         ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair),
                          mode='a', complevel=9, complib='blosc')
         ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS),

freqtrade/data/history/history_utils.py

@@ -9,7 +9,8 @@ from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
-from freqtrade.data.converter import (ohlcv_to_dataframe,
+from freqtrade.data.converter import (clean_ohlcv_dataframe,
+                                      ohlcv_to_dataframe,
                                       trades_remove_duplicates,
                                       trades_to_ohlcv)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
@@ -202,7 +203,10 @@ def _download_pair_history(datadir: Path,
         if data.empty:
             data = new_dataframe
         else:
-            data = data.append(new_dataframe)
+            # Run cleaning again to ensure there were no duplicate candles
+            # Especially between existing and new data.
+            data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair,
+                                         fill_missing=False, drop_incomplete=False)
 
         logger.debug("New  Start: %s",
                      f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')