adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2372 from xmatthias/kraken_ohlcv_emulate

Browse Source 
download tick-based data to emulate candles
This commit is contained in:
hroff-1902
2019-10-19 19:32:37 +03:00
committed by GitHub
parent 4c977b2e01 fd22c87295
commit 47fabca1d9
19 changed files with 799 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
tests/conftest.py
View File

@@ -930,6 +930,110 @@ def trades_for_order():
'fee': {'cost': 0.008, 'currency': 'LTC'}}]
@pytest.fixture(scope="function")
def trades_history():
return [{'info': {'a': 126181329,
'p': '0.01962700',
'q': '0.04000000',
'f': 138604155,
'l': 138604155,
'T': 1565798399463,
'm': False,
'M': True},
'timestamp': 1565798399463,
'datetime': '2019-08-14T15:59:59.463Z',
'symbol': 'ETH/BTC',
'id': '126181329',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'buy',
'price': 0.019627,
'amount': 0.04,
'cost': 0.00078508,
'fee': None},
{'info': {'a': 126181330,
'p': '0.01962700',
'q': '0.24400000',
'f': 138604156,
'l': 138604156,
'T': 1565798399629,
'm': False,
'M': True},
'timestamp': 1565798399629,
'datetime': '2019-08-14T15:59:59.629Z',
'symbol': 'ETH/BTC',
'id': '126181330',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'buy',
'price': 0.019627,
'amount': 0.244,
'cost': 0.004788987999999999,
'fee': None},
{'info': {'a': 126181331,
'p': '0.01962600',
'q': '0.01100000',
'f': 138604157,
'l': 138604157,
'T': 1565798399752,
'm': True,
'M': True},
'timestamp': 1565798399752,
'datetime': '2019-08-14T15:59:59.752Z',
'symbol': 'ETH/BTC',
'id': '126181331',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'sell',
'price': 0.019626,
'amount': 0.011,
'cost': 0.00021588599999999999,
'fee': None},
{'info': {'a': 126181332,
'p': '0.01962600',
'q': '0.01100000',
'f': 138604158,
'l': 138604158,
'T': 1565798399862,
'm': True,
'M': True},
'timestamp': 1565798399862,
'datetime': '2019-08-14T15:59:59.862Z',
'symbol': 'ETH/BTC',
'id': '126181332',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'sell',
'price': 0.019626,
'amount': 0.011,
'cost': 0.00021588599999999999,
'fee': None},
{'info': {'a': 126181333,
'p': '0.01952600',
'q': '0.01200000',
'f': 138604158,
'l': 138604158,
'T': 1565798399872,
'm': True,
'M': True},
'timestamp': 1565798399872,
'datetime': '2019-08-14T15:59:59.872Z',
'symbol': 'ETH/BTC',
'id': '126181333',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'sell',
'price': 0.019626,
'amount': 0.011,
'cost': 0.00021588599999999999,
'fee': None}]
@pytest.fixture(scope="function")
def trades_for_order2():
return [{'info': {'id': 34567,

114
tests/data/test_history.py
View File

@@ -13,15 +13,20 @@ from pandas import DataFrame
from freqtrade import OperationalException
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.data.history import (download_pair_history,
_load_cached_data_for_updating,
load_tickerdata_file,
from freqtrade.data.history import (_load_cached_data_for_updating,
convert_trades_to_ohlcv,
download_pair_history,
download_trades_history,
load_tickerdata_file, pair_data_filename,
pair_trades_filename,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data,
trim_tickerlist)
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json
from freqtrade.strategy.default_strategy import DefaultStrategy
from tests.conftest import get_patched_exchange, log_has, log_has_re, patch_exchange
from tests.conftest import (get_patched_exchange, log_has, log_has_re,
patch_exchange)
# Change this if modifying UNITTEST/BTC testdatafile
_BTC_UNITTEST_LENGTH = 13681
@@ -134,6 +139,18 @@ def test_testdata_path(testdatadir) -> None:
assert str(Path('tests') / 'testdata') in str(testdatadir)
def test_pair_data_filename():
fn = pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m')
assert isinstance(fn, Path)
assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json')
def test_pair_trades_filename():
fn = pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC')
assert isinstance(fn, Path)
assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz')
def test_load_cached_data_for_updating(mocker) -> None:
datadir = Path(__file__).parent.parent.joinpath('testdata')
@@ -569,3 +586,92 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
assert "ETH/BTC" in unav_pairs
assert "XRP/BTC" in unav_pairs
assert log_has("Skipping pair ETH/BTC...", caplog)
def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102")
unavailable_pairs = refresh_backtest_trades_data(exchange=ex,
pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"],
datadir=testdatadir,
timerange=timerange, erase=True
)
assert dl_mock.call_count == 2
assert dl_mock.call_args[1]['timerange'].starttype == 'date'
assert log_has("Downloading trades for pair ETH/BTC.", caplog)
assert unavailable_pairs == ["XRP/ETH"]
assert log_has("Skipping pair XRP/ETH...", caplog)
def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog) -> None:
ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history))
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
ght_mock)
exchange = get_patched_exchange(mocker, default_conf)
file1 = testdatadir / 'ETH_BTC-trades.json.gz'
_backup_file(file1)
assert not file1.is_file()
assert download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert log_has("New Amount of trades: 5", caplog)
assert file1.is_file()
# clean files freshly downloaded
_clean_test_file(file1)
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
MagicMock(side_effect=ValueError))
assert not download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog):
pair = 'XRP/ETH'
file1 = testdatadir / 'XRP_ETH-1m.json'
file5 = testdatadir / 'XRP_ETH-5m.json'
# Compare downloaded dataset with converted dataset
dfbak_1m = history.load_pair_history(datadir=testdatadir,
ticker_interval="1m",
pair=pair)
dfbak_5m = history.load_pair_history(datadir=testdatadir,
ticker_interval="5m",
pair=pair)
_backup_file(file1, copy_file=True)
_backup_file(file5)
tr = TimeRange.parse_timerange('20191011-20191012')
convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
datadir=testdatadir, timerange=tr, erase=True)
assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
# Load new data
df_1m = history.load_pair_history(datadir=testdatadir,
ticker_interval="1m",
pair=pair)
df_5m = history.load_pair_history(datadir=testdatadir,
ticker_interval="5m",
pair=pair)
assert df_1m.equals(dfbak_1m)
assert df_5m.equals(dfbak_5m)
_clean_test_file(file1)
_clean_test_file(file5)

203
tests/exchange/test_exchange.py
View File

@@ -1142,6 +1142,13 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_
await exchange._async_get_candle_history(pair, "5m",
(arrow.utcnow().timestamp - 2000) * 1000)
with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
r'historical candlestick data\..*'):
api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
await exchange._async_get_candle_history(pair, "5m",
(arrow.utcnow().timestamp - 2000) * 1000)
@pytest.mark.asyncio
async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
@@ -1313,6 +1320,196 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na
assert ticks[9][5] == 2.31452783
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
trades_history):
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
# Monkey-patch async function
exchange._api_async.fetch_trades = get_mock_coro(trades_history)
pair = 'ETH/BTC'
res = await exchange._async_fetch_trades(pair, since=None, params=None)
assert type(res) is list
assert isinstance(res[0], dict)
assert isinstance(res[1], dict)
assert exchange._api_async.fetch_trades.call_count == 1
assert exchange._api_async.fetch_trades.call_args[0][0] == pair
assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
assert log_has_re(f"Fetching trades for pair {pair}, since .*", caplog)
caplog.clear()
exchange._api_async.fetch_trades.reset_mock()
res = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'})
assert exchange._api_async.fetch_trades.call_count == 1
assert exchange._api_async.fetch_trades.call_args[0][0] == pair
assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
assert exchange._api_async.fetch_trades.call_args[1]['params'] == {'from': '123'}
assert log_has_re(f"Fetching trades for pair {pair}, params: .*", caplog)
exchange = Exchange(default_conf)
await async_ccxt_exception(mocker, default_conf, MagicMock(),
"_async_fetch_trades", "fetch_trades",
pair='ABCD/BTC', since=None)
api_mock = MagicMock()
with pytest.raises(OperationalException, match=r'Could not fetch trade data*'):
api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
r'historical trade data\..*'):
api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchange_name,
trades_history):
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
pagination_arg = exchange._trades_pagination_arg
async def mock_get_trade_hist(pair, *args, **kwargs):
if 'since' in kwargs:
# Return first 3
return trades_history[:-2]
elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']:
# Return 2
return trades_history[-3:-1]
else:
# Return last 2
return trades_history[-2:]
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
assert len(ret[1]) == len(trades_history)
assert exchange._async_fetch_trades.call_count == 3
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
# 2nd call
assert fetch_trades_cal[1][0][0] == pair
assert 'params' in fetch_trades_cal[1][1]
assert exchange._ft_has['trades_pagination_arg'] in fetch_trades_cal[1][1]['params']
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_get_trade_history_time(default_conf, mocker, caplog, exchange_name,
trades_history):
caplog.set_level(logging.DEBUG)
async def mock_get_trade_hist(pair, *args, **kwargs):
if kwargs['since'] == trades_history[0]["timestamp"]:
return trades_history[:-1]
else:
return trades_history[-1:]
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
assert len(ret[1]) == len(trades_history)
assert exchange._async_fetch_trades.call_count == 2
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
# 2nd call
assert fetch_trades_cal[1][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
assert log_has_re(r"Stopping because until was reached.*", caplog)
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, exchange_name,
trades_history):
caplog.set_level(logging.DEBUG)
async def mock_get_trade_hist(pair, *args, **kwargs):
if kwargs['since'] == trades_history[0]["timestamp"]:
return trades_history[:-1]
else:
return []
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
assert len(ret[1]) == len(trades_history) - 1
assert exchange._async_fetch_trades.call_count == 2
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades_history):
mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
pair = 'ETH/BTC'
exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history))
exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history))
ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"])
# Depending on the exchange, one or the other method should be called
assert sum([exchange._async_get_trade_history_id.call_count,
exchange._async_get_trade_history_time.call_count]) == 1
assert len(ret) == 2
assert ret[0] == pair
assert len(ret[1]) == len(trades_history)
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange_name,
trades_history):
mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=False)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
pair = 'ETH/BTC'
with pytest.raises(OperationalException,
match="This exchange does not suport downloading Trades."):
exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"])
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
default_conf['dry_run'] = True
@@ -1459,13 +1656,17 @@ def test_merge_ft_has_dict(default_conf, mocker):
assert ex._ft_has == Exchange._ft_has_default
ex = Kraken(default_conf)
assert ex._ft_has == Exchange._ft_has_default
assert ex._ft_has != Exchange._ft_has_default
assert ex._ft_has['trades_pagination'] == 'id'
assert ex._ft_has['trades_pagination_arg'] == 'since'
# Binance defines different values
ex = Binance(default_conf)
assert ex._ft_has != Exchange._ft_has_default
assert ex._ft_has['stoploss_on_exchange']
assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
assert ex._ft_has['trades_pagination'] == 'id'
assert ex._ft_has['trades_pagination_arg'] == 'fromId'
conf = copy.deepcopy(default_conf)
conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,

22
tests/test_utils.py
View File

@@ -260,3 +260,25 @@ def test_download_data_no_pairs(mocker, caplog):
with pytest.raises(OperationalException,
match=r"Downloading data requires a list of pairs\..*"):
start_download_data(pargs)
def test_download_data_trades(mocker, caplog):
dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data',
MagicMock(return_value=[]))
convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv',
MagicMock(return_value=[]))
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
args = [
"download-data",
"--exchange", "kraken",
"--pairs", "ETH/BTC", "XRP/BTC",
"--days", "20",
"--dl-trades"
]
start_download_data(get_args(args))
assert dl_mock.call_args[1]['timerange'].starttype == "date"
assert dl_mock.call_count == 1
assert convert_mock.call_count == 1

1
tests/testdata/XRP_ETH-1m.json vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
tests/testdata/XRP_ETH-5m.json vendored Normal file
View File

File diff suppressed because one or more lines are too long

BIN
tests/testdata/XRP_ETH-trades.json.gz vendored Normal file
View File

Binary file not shown.