adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3129 from freqtrade/trades_to_list

Browse Source 
convert dl-trades datadownload to list
This commit is contained in:
Matthias
2020-05-13 09:41:26 +02:00
committed by GitHub
parent ed7209f910 7aba9bc62a
commit d86855f2f3
12 changed files with 221 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tests/conftest.py
View File

@@ -1422,6 +1422,15 @@ def trades_for_order():
@pytest.fixture(scope="function")
def trades_history():
return [[1565798399463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508],
[1565798399629, '126181330', None, 'buy', 0.019627, 0.244, 0.004788987999999999],
[1565798399752, '126181331', None, 'sell', 0.019626, 0.011, 0.00021588599999999999],
[1565798399862, '126181332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999],
[1565798399872, '126181333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]]
@pytest.fixture(scope="function")
def fetch_trades_result():
return [{'info': {'a': 126181329,
'p': '0.01962700',
'q': '0.04000000',

70
tests/data/test_converter.py
View File

@@ -5,12 +5,10 @@ from freqtrade.configuration.timerange import TimeRange
from freqtrade.data.converter import (convert_ohlcv_format,
convert_trades_format,
ohlcv_fill_up_missing_data,
ohlcv_to_dataframe,
trim_dataframe)
from freqtrade.data.history import (get_timerange,
load_data,
load_pair_history,
validate_backtest_data)
ohlcv_to_dataframe, trades_dict_to_list,
trades_remove_duplicates, trim_dataframe)
from freqtrade.data.history import (get_timerange, load_data,
load_pair_history, validate_backtest_data)
from tests.conftest import log_has
from tests.data.test_history import _backup_file, _clean_test_file
@@ -197,32 +195,60 @@ def test_trim_dataframe(testdatadir) -> None:
assert all(data_modify.iloc[0] == data.iloc[25])
def test_convert_trades_format(mocker, default_conf, testdatadir):
file = testdatadir / "XRP_ETH-trades.json.gz"
file_new = testdatadir / "XRP_ETH-trades.json"
_backup_file(file, copy_file=True)
default_conf['datadir'] = testdatadir
def test_trades_remove_duplicates(trades_history):
trades_history1 = trades_history * 3
assert len(trades_history1) == len(trades_history) * 3
res = trades_remove_duplicates(trades_history1)
assert len(res) == len(trades_history)
for i, t in enumerate(res):
assert t == trades_history[i]
assert not file_new.exists()
def test_trades_dict_to_list(fetch_trades_result):
res = trades_dict_to_list(fetch_trades_result)
assert isinstance(res, list)
assert isinstance(res[0], list)
for i, t in enumerate(res):
assert t[0] == fetch_trades_result[i]['timestamp']
assert t[1] == fetch_trades_result[i]['id']
assert t[2] == fetch_trades_result[i]['type']
assert t[3] == fetch_trades_result[i]['side']
assert t[4] == fetch_trades_result[i]['price']
assert t[5] == fetch_trades_result[i]['amount']
assert t[6] == fetch_trades_result[i]['cost']
def test_convert_trades_format(mocker, default_conf, testdatadir):
files = [{'old': testdatadir / "XRP_ETH-trades.json.gz",
'new': testdatadir / "XRP_ETH-trades.json"},
{'old': testdatadir / "XRP_OLD-trades.json.gz",
'new': testdatadir / "XRP_OLD-trades.json"},
]
for file in files:
_backup_file(file['old'], copy_file=True)
assert not file['new'].exists()
default_conf['datadir'] = testdatadir
convert_trades_format(default_conf, convert_from='jsongz',
convert_to='json', erase=False)
assert file_new.exists()
assert file.exists()
for file in files:
assert file['new'].exists()
assert file['old'].exists()
# Remove original file
file.unlink()
# Remove original file
file['old'].unlink()
# Convert back
convert_trades_format(default_conf, convert_from='json',
convert_to='jsongz', erase=True)
for file in files:
assert file['old'].exists()
assert not file['new'].exists()
assert file.exists()
assert not file_new.exists()
_clean_test_file(file)
if file_new.exists():
file_new.unlink()
_clean_test_file(file['old'])
if file['new'].exists():
file['new'].unlink()
def test_convert_ohlcv_format(mocker, default_conf, testdatadir):

24
tests/data/test_history.py
View File

@@ -547,6 +547,17 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
assert log_has("New Amount of trades: 5", caplog)
assert file1.is_file()
ght_mock.reset_mock()
since_time = int(trades_history[-3][0] // 1000)
since_time2 = int(trades_history[-1][0] // 1000)
timerange = TimeRange('date', None, since_time, 0)
assert _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='ETH/BTC', timerange=timerange)
assert ght_mock.call_count == 1
# Check this in seconds - since we had to convert to seconds above too.
assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5
# clean files freshly downloaded
_clean_test_file(file1)
@@ -601,7 +612,7 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
def test_jsondatahandler_trades_get_pairs(testdatadir):
pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
# Convert to set to avoid failures due to sorting
assert set(pairs) == {'XRP/ETH'}
assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
@@ -614,6 +625,17 @@ def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m')
def test_jsondatahandler_trades_load(mocker, testdatadir, caplog):
dh = JsonGzDataHandler(testdatadir)
logmsg = "Old trades format detected - converting"
dh.trades_load('XRP/ETH')
assert not log_has(logmsg, caplog)
# Test conversation is happening
dh.trades_load('XRP/OLD')
assert log_has(logmsg, caplog)
def test_jsondatahandler_trades_purge(mocker, testdatadir):
mocker.patch.object(Path, "exists", MagicMock(return_value=False))
mocker.patch.object(Path, "unlink", MagicMock())

42
tests/exchange/test_exchange.py
View File

@@ -1537,18 +1537,18 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
trades_history):
fetch_trades_result):
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
# Monkey-patch async function
exchange._api_async.fetch_trades = get_mock_coro(trades_history)
exchange._api_async.fetch_trades = get_mock_coro(fetch_trades_result)
pair = 'ETH/BTC'
res = await exchange._async_fetch_trades(pair, since=None, params=None)
assert type(res) is list
assert isinstance(res[0], dict)
assert isinstance(res[1], dict)
assert isinstance(res[0], list)
assert isinstance(res[1], list)
assert exchange._api_async.fetch_trades.call_count == 1
assert exchange._api_async.fetch_trades.call_args[0][0] == pair
@@ -1594,7 +1594,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
if 'since' in kwargs:
# Return first 3
return trades_history[:-2]
elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']:
elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3][1]:
# Return 2
return trades_history[-3:-1]
else:
@@ -1604,8 +1604,8 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0][0],
until=trades_history[-1][0]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
@@ -1614,7 +1614,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
# 2nd call
assert fetch_trades_cal[1][0][0] == pair
@@ -1630,7 +1630,7 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
caplog.set_level(logging.DEBUG)
async def mock_get_trade_hist(pair, *args, **kwargs):
if kwargs['since'] == trades_history[0]["timestamp"]:
if kwargs['since'] == trades_history[0][0]:
return trades_history[:-1]
else:
return trades_history[-1:]
@@ -1640,8 +1640,8 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0],
until=trades_history[-1][0]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
@@ -1650,11 +1650,11 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
# 2nd call
assert fetch_trades_cal[1][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
assert log_has_re(r"Stopping because until was reached.*", caplog)
@@ -1666,7 +1666,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
caplog.set_level(logging.DEBUG)
async def mock_get_trade_hist(pair, *args, **kwargs):
if kwargs['since'] == trades_history[0]["timestamp"]:
if kwargs['since'] == trades_history[0][0]:
return trades_history[:-1]
else:
return []
@@ -1676,8 +1676,8 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0],
until=trades_history[-1][0]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
@@ -1686,7 +1686,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
@pytest.mark.parametrize("exchange_name", EXCHANGES)
@@ -1698,8 +1698,8 @@ def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades
exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history))
exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history))
ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"])
ret = exchange.get_historic_trades(pair, since=trades_history[0][0],
until=trades_history[-1][0])
# Depending on the exchange, one or the other method should be called
assert sum([exchange._async_get_trade_history_id.call_count,
@@ -1720,8 +1720,8 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange
with pytest.raises(OperationalException,
match="This exchange does not suport downloading Trades."):
exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"])
exchange.get_historic_trades(pair, since=trades_history[0][0],
until=trades_history[-1][0])
@pytest.mark.parametrize("exchange_name", EXCHANGES)

BIN
tests/testdata/XRP_ETH-trades.json.gz vendored
View File

Binary file not shown.

BIN
tests/testdata/XRP_OLD-trades.json.gz vendored Normal file
View File

Binary file not shown.