Merge pull request #3971 from freqtrade/fix/3967

Fix bug when converting trades do ohlcv and no trades are available.

freqtrade/constants.py

@@ -365,3 +365,6 @@ CANCEL_REASON = {
 # List of pairs with their timeframes
 PairWithTimeframe = Tuple[str, str]
 ListPairsWithTimeframes = List[PairWithTimeframe]
+
+# Type for trades list
+TradeList = List[List]

freqtrade/data/converter.py

@@ -10,7 +10,7 @@ from typing import Any, Dict, List
 import pandas as pd
 from pandas import DataFrame, to_datetime
 
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
 
 
 logger = logging.getLogger(__name__)
@@ -168,7 +168,7 @@ def trades_remove_duplicates(trades: List[List]) -> List[List]:
     return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))]
 
 
-def trades_dict_to_list(trades: List[Dict]) -> List[List]:
+def trades_dict_to_list(trades: List[Dict]) -> TradeList:
     """
     Convert fetch_trades result into a List (to be more memory efficient).
     :param trades: List of trades, as returned by ccxt.fetch_trades.
@@ -177,16 +177,18 @@ def trades_dict_to_list(trades: List[Dict]) -> List[List]:
     return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades]
 
 
-def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame:
+def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame:
     """
     Converts trades list to OHLCV list
-    TODO: This should get a dedicated test
     :param trades: List of trades, as returned by ccxt.fetch_trades.
     :param timeframe: Timeframe to resample data to
     :return: OHLCV Dataframe.
+    :raises: ValueError if no trades are provided
     """
     from freqtrade.exchange import timeframe_to_minutes
     timeframe_minutes = timeframe_to_minutes(timeframe)
+    if not trades:
+        raise ValueError('Trade-list empty.')
     df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS)
     df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms',
                                      utc=True,)

freqtrade/data/history/hdf5datahandler.py

@@ -9,9 +9,9 @@ import pandas as pd
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS,
-                                 ListPairsWithTimeframes)
+                                 ListPairsWithTimeframes, TradeList)
 
-from .idatahandler import IDataHandler, TradeList
+from .idatahandler import IDataHandler
 
 
 logger = logging.getLogger(__name__)

freqtrade/data/history/history_utils.py

@@ -356,9 +356,12 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
             if erase:
                 if data_handler_ohlcv.ohlcv_purge(pair, timeframe):
                     logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
-            ohlcv = trades_to_ohlcv(trades, timeframe)
+            try:
-            # Store ohlcv
+                ohlcv = trades_to_ohlcv(trades, timeframe)
-            data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv)
+                # Store ohlcv
+                data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv)
+            except ValueError:
+                logger.exception(f'Could not convert {pair} to OHLCV.')
 
 
 def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:

freqtrade/data/history/idatahandler.py

@@ -13,16 +13,13 @@ from typing import List, Optional, Type
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import ListPairsWithTimeframes
+from freqtrade.constants import ListPairsWithTimeframes, TradeList
 from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe
 from freqtrade.exchange import timeframe_to_seconds
 
 
 logger = logging.getLogger(__name__)
 
-# Type for trades list
-TradeList = List[List]
-
 
 class IDataHandler(ABC):
 

freqtrade/data/history/jsondatahandler.py

@@ -8,10 +8,10 @@ from pandas import DataFrame, read_json, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList
 from freqtrade.data.converter import trades_dict_to_list
 
-from .idatahandler import IDataHandler, TradeList
+from .idatahandler import IDataHandler
 
 
 logger = logging.getLogger(__name__)

tests/data/test_converter.py

@@ -1,10 +1,13 @@
 # pragma pylint: disable=missing-docstring, C0103
 import logging
 
+import pytest
+
 from freqtrade.configuration.timerange import TimeRange
 from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format,
                                       ohlcv_fill_up_missing_data, ohlcv_to_dataframe,
-                                      trades_dict_to_list, trades_remove_duplicates, trim_dataframe)
+                                      trades_dict_to_list, trades_remove_duplicates,
+                                      trades_to_ohlcv, trim_dataframe)
 from freqtrade.data.history import (get_timerange, load_data, load_pair_history,
                                     validate_backtest_data)
 from tests.conftest import log_has
@@ -26,6 +29,28 @@ def test_ohlcv_to_dataframe(ohlcv_history_list, caplog):
     assert log_has('Converting candle (OHLCV) data to dataframe for pair UNITTEST/BTC.', caplog)
 
 
+def test_trades_to_ohlcv(ohlcv_history_list, caplog):
+
+    caplog.set_level(logging.DEBUG)
+    with pytest.raises(ValueError, match="Trade-list empty."):
+        trades_to_ohlcv([], '1m')
+
+    trades = [
+        [1570752011620, "13519807", None, "sell", 0.00141342, 23.0, 0.03250866],
+        [1570752011620, "13519808", None, "sell", 0.00141266, 54.0, 0.07628364],
+        [1570752017964, "13519809", None, "sell", 0.00141266, 8.0, 0.01130128]]
+
+    df = trades_to_ohlcv(trades, '1m')
+    assert not df.empty
+    assert len(df) == 1
+    assert 'open' in df.columns
+    assert 'high' in df.columns
+    assert 'low' in df.columns
+    assert 'close' in df.columns
+    assert df.loc[:, 'high'][0] == 0.00141342
+    assert df.loc[:, 'low'][0] == 0.00141266
+
+
 def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
     data = load_pair_history(datadir=testdatadir,
                              timeframe='1m',

tests/data/test_history.py

@@ -620,6 +620,12 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog):
     _clean_test_file(file1)
     _clean_test_file(file5)
 
+    assert not log_has('Could not convert NoDatapair to OHLCV.', caplog)
+
+    convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
+                            datadir=testdatadir, timerange=tr, erase=True)
+    assert log_has('Could not convert NoDatapair to OHLCV.', caplog)
+
 
 def test_datahandler_ohlcv_get_pairs(testdatadir):
     pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m')