diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0a35ab1fe..34e1c63f1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,7 +24,9 @@ AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] -DEFAULT_TRADES_COLUMNS = ['timestamp', 'type', 'side', 'price', 'amount', 'cost'] +# Don't modify sequence of DEFAULT_TRADES_COLUMNS +# it has wide consequences for stored trades files +DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGIES = 'strategies' diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 77371bf27..7abd0453f 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -3,12 +3,13 @@ Functions to convert data from one format to another """ import logging from datetime import datetime, timezone -from typing import Any, Dict +from typing import Any, Dict, List import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS + logger = logging.getLogger(__name__) @@ -154,7 +155,16 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: return frame -def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: +def trades_dict_to_list(trades: List[Dict]) -> List[List]: + """ + Convert fetch_trades result into a List (to be more memory efficient). + :param trades: List of trades, as returned by ccxt.fetch_trades. + :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns + """ + return [[t[col] for col in DEFAULT_TRADES_COLUMNS]for t in trades] + + +def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame: """ Converts trades list to OHLCV list TODO: This should get a dedicated test diff --git a/tests/conftest.py b/tests/conftest.py index 64d0cd5ee..72f9bfe46 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1391,6 +1391,50 @@ def buy_order_fee(): } +@pytest.fixture +def fetch_trades_result(): + return [{'info': {'a': 153942751, + 'p': '0.02059000', + 'q': '6.95500000', + 'f': 170811204, + 'l': 170811204, + 'T': 1585670882666, + 'm': True, + 'M': True}, + 'timestamp': 1585670882666, + 'datetime': '2020-03-31T16:08:02.666Z', + 'symbol': 'ETH/BTC', + 'id': '153942751', + 'order': None, + 'type': None, + 'takerOrMaker': None, + 'side': 'sell', + 'price': 0.02059, + 'amount': 6.955, + 'cost': 0.14320345, + 'fee': None}, + {'info': {'a': 153942752, + 'p': '0.02059100', + 'q': '0.04900000', + 'f': 170811205, + 'l': 170811205, + 'T': 1585670889154, + 'm': False, + 'M': True}, + 'timestamp': 1585670889154, + 'datetime': '2020-03-31T16:08:09.154Z', + 'symbol': 'ETH/BTC', + 'id': '153942752', + 'order': None, + 'type': None, + 'takerOrMaker': None, + 'side': 'buy', + 'price': 0.020591, + 'amount': 0.049, + 'cost': 0.001008959, + 'fee': None}] + + @pytest.fixture(scope="function") def edge_conf(default_conf): conf = deepcopy(default_conf) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 7dff520e0..1f7837e46 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -5,12 +5,10 @@ from freqtrade.configuration.timerange import TimeRange from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, ohlcv_fill_up_missing_data, - ohlcv_to_dataframe, + ohlcv_to_dataframe, trades_dict_to_list, trim_dataframe) -from freqtrade.data.history import (get_timerange, - load_data, - load_pair_history, - validate_backtest_data) +from freqtrade.data.history import (get_timerange, load_data, + load_pair_history, validate_backtest_data) from tests.conftest import log_has from tests.data.test_history import _backup_file, _clean_test_file @@ -197,6 +195,21 @@ def test_trim_dataframe(testdatadir) -> None: assert all(data_modify.iloc[0] == data.iloc[25]) +def test_trades_dict_to_list(mocker, fetch_trades_result): + res = trades_dict_to_list(fetch_trades_result) + assert isinstance(res, list) + assert isinstance(res[0], list) + for i, t in enumerate(res): + assert t[0] == fetch_trades_result[i]['timestamp'] + assert t[1] == fetch_trades_result[i]['id'] + assert t[2] == fetch_trades_result[i]['type'] + assert t[3] == fetch_trades_result[i]['side'] + assert t[4] == fetch_trades_result[i]['price'] + assert t[5] == fetch_trades_result[i]['amount'] + assert t[6] == fetch_trades_result[i]['cost'] + + + def test_convert_trades_format(mocker, default_conf, testdatadir): file = testdatadir / "XRP_ETH-trades.json.gz" file_new = testdatadir / "XRP_ETH-trades.json"