Merge pull request #8361 from TheJoeSchr/feature/trades-feather

featherdatahandler: implement trades_store/_trades_load
This commit is contained in:
Matthias 2023-03-27 21:05:30 +02:00 committed by GitHub
commit 8ae44c204e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 67 additions and 16 deletions

View File

@ -36,9 +36,10 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', '
'AgeFilter', 'OffsetFilter', 'PerformanceFilter', 'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_PROTECTIONS = ['CooldownPeriod',
AVAILABLE_DATAHANDLERS_TRADES = ['json', 'jsongz', 'hdf5'] 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
AVAILABLE_DATAHANDLERS = AVAILABLE_DATAHANDLERS_TRADES + ['feather', 'parquet'] AVAILABLE_DATAHANDLERS_TRADES = ['json', 'jsongz', 'hdf5', 'feather']
AVAILABLE_DATAHANDLERS = AVAILABLE_DATAHANDLERS_TRADES + ['parquet']
BACKTEST_BREAKDOWNS = ['day', 'week', 'month'] BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month'] BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month']
BACKTEST_CACHE_DEFAULT = 'day' BACKTEST_CACHE_DEFAULT = 'day'

View File

@ -4,7 +4,7 @@ from typing import Optional
from pandas import DataFrame, read_feather, to_datetime from pandas import DataFrame, read_feather, to_datetime
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
from freqtrade.enums import CandleType from freqtrade.enums import CandleType
from .idatahandler import IDataHandler from .idatahandler import IDataHandler
@ -92,12 +92,11 @@ class FeatherDataHandler(IDataHandler):
:param data: List of Lists containing trade data, :param data: List of Lists containing trade data,
column sequence as in DEFAULT_TRADES_COLUMNS column sequence as in DEFAULT_TRADES_COLUMNS
""" """
# filename = self._pair_trades_filename(self._datadir, pair) filename = self._pair_trades_filename(self._datadir, pair)
self.create_dir_if_needed(filename)
raise NotImplementedError() tradesdata = DataFrame(data, columns=DEFAULT_TRADES_COLUMNS)
# array = pa.array(data) tradesdata.to_feather(filename, compression_level=9, compression='lz4')
# array
# feather.write_feather(data, filename)
def trades_append(self, pair: str, data: TradeList): def trades_append(self, pair: str, data: TradeList):
""" """
@ -116,14 +115,13 @@ class FeatherDataHandler(IDataHandler):
:param timerange: Timerange to load trades for - currently not implemented :param timerange: Timerange to load trades for - currently not implemented
:return: List of trades :return: List of trades
""" """
raise NotImplementedError() filename = self._pair_trades_filename(self._datadir, pair)
# filename = self._pair_trades_filename(self._datadir, pair) if not filename.exists():
# tradesdata = misc.file_load_json(filename) return []
# if not tradesdata: tradesdata = read_feather(filename)
# return []
# return tradesdata return tradesdata.values.tolist()
@classmethod @classmethod
def _get_file_extension(cls): def _get_file_extension(cls):

View File

@ -252,7 +252,7 @@ def test_datahandler__check_empty_df(testdatadir, caplog):
assert log_has_re(expected_text, caplog) assert log_has_re(expected_text, caplog)
@pytest.mark.parametrize('datahandler', ['feather', 'parquet']) @pytest.mark.parametrize('datahandler', ['parquet'])
def test_datahandler_trades_not_supported(datahandler, testdatadir, ): def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
dh = get_datahandler(testdatadir, datahandler) dh = get_datahandler(testdatadir, datahandler)
with pytest.raises(NotImplementedError): with pytest.raises(NotImplementedError):
@ -496,6 +496,58 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
assert unlinkmock.call_count == 2 assert unlinkmock.call_count == 2
def test_featherdatahandler_trades_load(testdatadir):
dh = get_datahandler(testdatadir, 'feather')
trades = dh.trades_load('XRP/ETH')
assert isinstance(trades, list)
assert trades[0][0] == 1570752011620
assert trades[-1][-1] == 0.1986231
trades1 = dh.trades_load('UNITTEST/NONEXIST')
assert trades1 == []
def test_featherdatahandler_trades_store(testdatadir, tmpdir):
tmpdir1 = Path(tmpdir)
dh = get_datahandler(testdatadir, 'feather')
trades = dh.trades_load('XRP/ETH')
dh1 = get_datahandler(tmpdir1, 'feather')
dh1.trades_store('XRP/NEW', trades)
file = tmpdir1 / 'XRP_NEW-trades.feather'
assert file.is_file()
# Load trades back
trades_new = dh1.trades_load('XRP/NEW')
assert len(trades_new) == len(trades)
assert trades[0][0] == trades_new[0][0]
assert trades[0][1] == trades_new[0][1]
# assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense
assert trades[0][3] == trades_new[0][3]
assert trades[0][4] == trades_new[0][4]
assert trades[0][5] == trades_new[0][5]
assert trades[0][6] == trades_new[0][6]
assert trades[-1][0] == trades_new[-1][0]
assert trades[-1][1] == trades_new[-1][1]
# assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense
assert trades[-1][3] == trades_new[-1][3]
assert trades[-1][4] == trades_new[-1][4]
assert trades[-1][5] == trades_new[-1][5]
assert trades[-1][6] == trades_new[-1][6]
def test_featherdatahandler_trades_purge(mocker, testdatadir):
mocker.patch.object(Path, "exists", MagicMock(return_value=False))
unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
dh = get_datahandler(testdatadir, 'feather')
assert not dh.trades_purge('UNITTEST/NONEXIST')
assert unlinkmock.call_count == 0
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
assert dh.trades_purge('UNITTEST/NONEXIST')
assert unlinkmock.call_count == 1
def test_gethandlerclass(): def test_gethandlerclass():
cl = get_datahandlerclass('json') cl = get_datahandlerclass('json')
assert cl == JsonDataHandler assert cl == JsonDataHandler

BIN
tests/testdata/XRP_ETH-trades.feather vendored Normal file

Binary file not shown.