Merge pull request #2372 from xmatthias/kraken_ohlcv_emulate

download tick-based data to emulate candles
This commit is contained in:
hroff-1902 2019-10-19 19:32:37 +03:00 committed by GitHub
commit 47fabca1d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 799 additions and 15 deletions

View File

@ -70,5 +70,6 @@
"forcebuy_enable": false,
"internals": {
"process_throttle_secs": 5
}
},
"download_trades": true
}

View File

@ -38,7 +38,7 @@ Mixing different stake-currencies is allowed for this file, since it's only used
]
```
### start download
### Start download
Then run:
@ -57,6 +57,32 @@ This will download ticker data for all the currency pairs you defined in `pairs.
- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers.
- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
### Trades (tick) data
By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `<pair>-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades, and resamples the data locally.
Example call:
```bash
freqtrade download-data --exchange binance --pairs XRP/ETH ETH/BTC --days 20 --dl-trades
```
!!! Note
While this method uses async calls, it will be slow, since it requires the result of the previous call to generate the next request to the exchange.
!!! Warning
The historic trades are not available during Freqtrade dry-run and live trade modes because all exchanges tested provide this data with a delay of few 100 candles, so it's not suitable for real-time trading.
### Historic Kraken data
The Kraken API does only provide 720 historic candles, which is sufficient for FreqTrade dry-run and live trade modes, but is a problem for backtesting.
To download data for the Kraken exchange, using `--dl-trades` is mandatory, otherwise the bot will download the same 720 candles over and over, and you'll not have enough backtest data.
## Next step
Great, you now have backtest data downloaded, so you can now start [backtesting](backtesting.md) your strategy.

View File

@ -35,7 +35,8 @@ ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"]
ARGS_CREATE_USERDIR = ["user_data_dir"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
"timeframes", "erase"]
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url",
"trade_source", "export", "exportfilename", "timerange", "ticker_interval"]

View File

@ -273,6 +273,12 @@ AVAILABLE_CLI_OPTIONS = {
type=check_int_positive,
metavar='INT',
),
"download_trades": Arg(
'--dl-trades',
help='Download trades instead of OHLCV data. The bot will resample trades to the '
'desired timeframe as specified as --timeframes/-t.',
action='store_true',
),
"exchange": Arg(
'--exchange',
help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). '

View File

@ -312,6 +312,8 @@ class Configuration:
self._args_to_config(config, argname='days',
logstring='Detected --days: {}')
self._args_to_config(config, argname='download_trades',
logstring='Detected --dl-trades: {}')
def _process_runmode(self, config: Dict[str, Any]) -> None:

View File

@ -114,3 +114,25 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum'])
# logger.info('order book %s', frame )
return frame
def trades_to_ohlcv(trades: list, timeframe: str) -> list:
"""
Converts trades list to ohlcv list
:param trades: List of trades, as returned by ccxt.fetch_trades.
:param timeframe: Ticker timeframe to resample data to
:return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv)
"""
from freqtrade.exchange import timeframe_to_minutes
ticker_minutes = timeframe_to_minutes(timeframe)
df = pd.DataFrame(trades)
df['datetime'] = pd.to_datetime(df['datetime'])
df = df.set_index('datetime')
df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
df_new['date'] = df_new.index.astype("int64") // 10 ** 6
# Drop 0 volume rows
df_new = df_new.dropna()
columns = ["date", "open", "high", "low", "close", "volume"]
return list(zip(*[df_new[x].values.tolist() for x in columns]))

View File

@ -17,7 +17,7 @@ from pandas import DataFrame
from freqtrade import OperationalException, misc
from freqtrade.configuration import TimeRange
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
from freqtrade.exchange import Exchange, timeframe_to_minutes
logger = logging.getLogger(__name__)
@ -82,6 +82,29 @@ def store_tickerdata_file(datadir: Path, pair: str,
misc.file_dump_json(filename, data, is_zip=is_zip)
def load_trades_file(datadir: Path, pair: str,
timerange: Optional[TimeRange] = None) -> List[Dict]:
"""
Load a pair from file, either .json.gz or .json
:return: tradelist or empty list if unsuccesful
"""
filename = pair_trades_filename(datadir, pair)
tradesdata = misc.file_load_json(filename)
if not tradesdata:
return []
return tradesdata
def store_trades_file(datadir: Path, pair: str,
data: list, is_zip: bool = True):
"""
Stores tickerdata to file
"""
filename = pair_trades_filename(datadir, pair)
misc.file_dump_json(filename, data, is_zip=is_zip)
def _validate_pairdata(pair, pairdata, timerange: TimeRange):
if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000:
logger.warning('Missing data at start for pair %s, data starts at %s',
@ -173,6 +196,12 @@ def pair_data_filename(datadir: Path, pair: str, ticker_interval: str) -> Path:
return filename
def pair_trades_filename(datadir: Path, pair: str) -> Path:
pair_s = pair.replace("/", "_")
filename = datadir.joinpath(f'{pair_s}-trades.json.gz')
return filename
def _load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str,
timerange: Optional[TimeRange]) -> Tuple[List[Any],
Optional[int]]:
@ -299,6 +328,92 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
return pairs_not_available
def download_trades_history(datadir: Path,
exchange: Exchange,
pair: str,
timerange: Optional[TimeRange] = None) -> bool:
"""
Download trade history from the exchange.
Appends to previously downloaded trades data.
"""
try:
since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
trades = load_trades_file(datadir, pair)
from_id = trades[-1]['id'] if trades else None
logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
new_trades = exchange.get_historic_trades(pair=pair,
since=since if since else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000,
# until=xxx,
from_id=from_id,
)
trades.extend(new_trades[1])
store_trades_file(datadir, pair, trades)
logger.debug("New Start: %s", trades[0]['datetime'])
logger.debug("New End: %s", trades[-1]['datetime'])
logger.info(f"New Amount of trades: {len(trades)}")
return True
except Exception as e:
logger.error(
f'Failed to download historic trades for pair: "{pair}". '
f'Error: {e}'
)
return False
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, erase=False) -> List[str]:
"""
Refresh stored trades data.
Used by freqtrade download-data
:return: Pairs not available
"""
pairs_not_available = []
for pair in pairs:
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
dl_file = pair_trades_filename(datadir, pair)
if erase and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}.')
dl_file.unlink()
logger.info(f'Downloading trades for pair {pair}.')
download_trades_history(datadir=datadir, exchange=exchange,
pair=pair,
timerange=timerange)
return pairs_not_available
def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
datadir: Path, timerange: TimeRange, erase=False) -> None:
"""
Convert stored trades data to ohlcv data
"""
for pair in pairs:
trades = load_trades_file(datadir, pair)
for timeframe in timeframes:
ohlcv_file = pair_data_filename(datadir, pair, timeframe)
if erase and ohlcv_file.exists():
logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
ohlcv_file.unlink()
ohlcv = trades_to_ohlcv(trades, timeframe)
# Store ohlcv
store_tickerdata_file(datadir, pair, timeframe, data=ohlcv)
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
"""
Get the maximum timeframe for the given backtest data

View File

@ -16,6 +16,8 @@ class Binance(Exchange):
_ft_has: Dict = {
"stoploss_on_exchange": True,
"order_time_in_force": ['gtc', 'fok', 'ioc'],
"trades_pagination": "id",
"trades_pagination_arg": "fromId",
}
def get_order_book(self, pair: str, limit: int = 100) -> dict:

View File

@ -147,6 +147,8 @@ def retrier(f):
class Exchange:
_config: Dict = {}
# Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
_params: Dict = {}
# Dict to specify which options each exchange implements
@ -157,6 +159,9 @@ class Exchange:
"order_time_in_force": ["gtc"],
"ohlcv_candle_limit": 500,
"ohlcv_partial_candle": True,
"trades_pagination": "time", # Possible are "time" or "id"
"trades_pagination_arg": "since",
}
_ft_has: Dict = {}
@ -200,6 +205,9 @@ class Exchange:
self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
self._trades_pagination = self._ft_has['trades_pagination']
self._trades_pagination_arg = self._ft_has['trades_pagination_arg']
# Initialize ccxt objects
self._api = self._init_ccxt(
exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config'))
@ -742,6 +750,154 @@ class Exchange:
except ccxt.BaseError as e:
raise OperationalException(f'Could not fetch ticker data. Msg: {e}') from e
@retrier_async
async def _async_fetch_trades(self, pair: str,
since: Optional[int] = None,
params: Optional[dict] = None) -> List[Dict]:
"""
Asyncronously gets trade history using fetch_trades.
Handles exchange errors, does one call to the exchange.
:param pair: Pair to fetch trade data for
:param since: Since as integer timestamp in milliseconds
returns: List of dicts containing trades
"""
try:
# fetch trades asynchronously
if params:
logger.debug("Fetching trades for pair %s, params: %s ", pair, params)
trades = await self._api_async.fetch_trades(pair, params=params, limit=1000)
else:
logger.debug(
"Fetching trades for pair %s, since %s %s...",
pair, since,
'(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
)
trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
return trades
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching historical trade data.'
f'Message: {e}') from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. '
f'Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e
async def _async_get_trade_history_id(self, pair: str,
until: int,
since: Optional[int] = None,
from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
"""
Asyncronously gets trade history using fetch_trades
use this when exchange uses id-based iteration (check `self._trades_pagination`)
:param pair: Pair to fetch trade data for
:param since: Since as integer timestamp in milliseconds
:param until: Until as integer timestamp in milliseconds
:param from_id: Download data starting with ID (if id is known). Ignores "since" if set.
returns tuple: (pair, trades-list)
"""
trades: List[Dict] = []
if not from_id:
# Fetch first elements using timebased method to get an ID to paginate on
# Depending on the Exchange, this can introduce a drift at the start of the interval
# of up to an hour.
# e.g. Binance returns the "last 1000" candles within a 1h time interval
# - so we will miss the first trades.
t = await self._async_fetch_trades(pair, since=since)
from_id = t[-1]['id']
trades.extend(t[:-1])
while True:
t = await self._async_fetch_trades(pair,
params={self._trades_pagination_arg: from_id})
if len(t):
# Skip last id since its the key for the next call
trades.extend(t[:-1])
if from_id == t[-1]['id'] or t[-1]['timestamp'] > until:
logger.debug(f"Stopping because from_id did not change. "
f"Reached {t[-1]['timestamp']} > {until}")
# Reached the end of the defined-download period - add last trade as well.
trades.extend(t[-1:])
break
from_id = t[-1]['id']
else:
break
return (pair, trades)
async def _async_get_trade_history_time(self, pair: str, until: int,
since: Optional[int] = None) -> Tuple[str, List]:
"""
Asyncronously gets trade history using fetch_trades,
when the exchange uses time-based iteration (check `self._trades_pagination`)
:param pair: Pair to fetch trade data for
:param since: Since as integer timestamp in milliseconds
:param until: Until as integer timestamp in milliseconds
returns tuple: (pair, trades-list)
"""
trades: List[Dict] = []
while True:
t = await self._async_fetch_trades(pair, since=since)
if len(t):
since = t[-1]['timestamp']
trades.extend(t)
# Reached the end of the defined-download period
if until and t[-1]['timestamp'] > until:
logger.debug(
f"Stopping because until was reached. {t[-1]['timestamp']} > {until}")
break
else:
break
return (pair, trades)
async def _async_get_trade_history(self, pair: str,
since: Optional[int] = None,
until: Optional[int] = None,
from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
"""
Async wrapper handling downloading trades using either time or id based methods.
"""
if self._trades_pagination == 'time':
return await self._async_get_trade_history_time(
pair=pair, since=since,
until=until or ccxt.Exchange.milliseconds())
elif self._trades_pagination == 'id':
return await self._async_get_trade_history_id(
pair=pair, since=since,
until=until or ccxt.Exchange.milliseconds(), from_id=from_id
)
else:
raise OperationalException(f"Exchange {self.name} does use neither time, "
f"nor id based pagination")
def get_historic_trades(self, pair: str,
since: Optional[int] = None,
until: Optional[int] = None,
from_id: Optional[str] = None) -> Tuple[str, List]:
"""
Gets candle history using asyncio and returns the list of candles.
Handles all async doing.
Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call.
:param pair: Pair to download
:param ticker_interval: Interval to get
:param since: Timestamp in milliseconds to get history from
:param until: Timestamp in milliseconds. Defaults to current timestamp if not defined.
:param from_id: Download data starting with ID (if id is known)
:returns List of tickers
"""
if not self.exchange_has("fetchTrades"):
raise OperationalException("This exchange does not suport downloading Trades.")
return asyncio.get_event_loop().run_until_complete(
self._async_get_trade_history(pair=pair, since=since,
until=until, from_id=from_id))
@retrier
def cancel_order(self, order_id: str, pair: str) -> None:
if self._config['dry_run']:

View File

@ -14,6 +14,10 @@ logger = logging.getLogger(__name__)
class Kraken(Exchange):
_params: Dict = {"trading_agreement": "agree"}
_ft_has: Dict = {
"trades_pagination": "id",
"trades_pagination_arg": "since",
}
@retrier
def get_balances(self) -> dict:

View File

@ -72,8 +72,10 @@ def json_load(datafile: IO):
def file_load_json(file):
gzipfile = file.with_suffix(file.suffix + '.gz')
if file.suffix != ".gz":
gzipfile = file.with_suffix(file.suffix + '.gz')
else:
gzipfile = file
# Try gzip file first, otherwise regular json file.
if gzipfile.is_file():
logger.debug('Loading ticker data from file %s', gzipfile)

View File

@ -8,7 +8,9 @@ import arrow
from freqtrade import OperationalException
from freqtrade.configuration import Configuration, TimeRange
from freqtrade.configuration.directory_operations import create_userdata_dir
from freqtrade.data.history import refresh_backtest_ohlcv_data
from freqtrade.data.history import (convert_trades_to_ohlcv,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data)
from freqtrade.exchange import available_exchanges, ccxt_exchanges
from freqtrade.resolvers import ExchangeResolver
from freqtrade.state import RunMode
@ -88,9 +90,19 @@ def start_download_data(args: Dict[str, Any]) -> None:
# Init exchange
exchange = ExchangeResolver(config['exchange']['name'], config).exchange
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config["pairs"], timeframes=config["timeframes"],
dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
if config.get('download_trades'):
pairs_not_available = refresh_backtest_trades_data(
exchange, pairs=config["pairs"], datadir=Path(config['datadir']),
timerange=timerange, erase=config.get("erase"))
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=config["pairs"], timeframes=config["timeframes"],
datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config["pairs"], timeframes=config["timeframes"],
dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")

View File

@ -930,6 +930,110 @@ def trades_for_order():
'fee': {'cost': 0.008, 'currency': 'LTC'}}]
@pytest.fixture(scope="function")
def trades_history():
return [{'info': {'a': 126181329,
'p': '0.01962700',
'q': '0.04000000',
'f': 138604155,
'l': 138604155,
'T': 1565798399463,
'm': False,
'M': True},
'timestamp': 1565798399463,
'datetime': '2019-08-14T15:59:59.463Z',
'symbol': 'ETH/BTC',
'id': '126181329',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'buy',
'price': 0.019627,
'amount': 0.04,
'cost': 0.00078508,
'fee': None},
{'info': {'a': 126181330,
'p': '0.01962700',
'q': '0.24400000',
'f': 138604156,
'l': 138604156,
'T': 1565798399629,
'm': False,
'M': True},
'timestamp': 1565798399629,
'datetime': '2019-08-14T15:59:59.629Z',
'symbol': 'ETH/BTC',
'id': '126181330',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'buy',
'price': 0.019627,
'amount': 0.244,
'cost': 0.004788987999999999,
'fee': None},
{'info': {'a': 126181331,
'p': '0.01962600',
'q': '0.01100000',
'f': 138604157,
'l': 138604157,
'T': 1565798399752,
'm': True,
'M': True},
'timestamp': 1565798399752,
'datetime': '2019-08-14T15:59:59.752Z',
'symbol': 'ETH/BTC',
'id': '126181331',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'sell',
'price': 0.019626,
'amount': 0.011,
'cost': 0.00021588599999999999,
'fee': None},
{'info': {'a': 126181332,
'p': '0.01962600',
'q': '0.01100000',
'f': 138604158,
'l': 138604158,
'T': 1565798399862,
'm': True,
'M': True},
'timestamp': 1565798399862,
'datetime': '2019-08-14T15:59:59.862Z',
'symbol': 'ETH/BTC',
'id': '126181332',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'sell',
'price': 0.019626,
'amount': 0.011,
'cost': 0.00021588599999999999,
'fee': None},
{'info': {'a': 126181333,
'p': '0.01952600',
'q': '0.01200000',
'f': 138604158,
'l': 138604158,
'T': 1565798399872,
'm': True,
'M': True},
'timestamp': 1565798399872,
'datetime': '2019-08-14T15:59:59.872Z',
'symbol': 'ETH/BTC',
'id': '126181333',
'order': None,
'type': None,
'takerOrMaker': None,
'side': 'sell',
'price': 0.019626,
'amount': 0.011,
'cost': 0.00021588599999999999,
'fee': None}]
@pytest.fixture(scope="function")
def trades_for_order2():
return [{'info': {'id': 34567,

View File

@ -13,15 +13,20 @@ from pandas import DataFrame
from freqtrade import OperationalException
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.data.history import (download_pair_history,
_load_cached_data_for_updating,
load_tickerdata_file,
from freqtrade.data.history import (_load_cached_data_for_updating,
convert_trades_to_ohlcv,
download_pair_history,
download_trades_history,
load_tickerdata_file, pair_data_filename,
pair_trades_filename,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data,
trim_tickerlist)
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json
from freqtrade.strategy.default_strategy import DefaultStrategy
from tests.conftest import get_patched_exchange, log_has, log_has_re, patch_exchange
from tests.conftest import (get_patched_exchange, log_has, log_has_re,
patch_exchange)
# Change this if modifying UNITTEST/BTC testdatafile
_BTC_UNITTEST_LENGTH = 13681
@ -134,6 +139,18 @@ def test_testdata_path(testdatadir) -> None:
assert str(Path('tests') / 'testdata') in str(testdatadir)
def test_pair_data_filename():
fn = pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m')
assert isinstance(fn, Path)
assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json')
def test_pair_trades_filename():
fn = pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC')
assert isinstance(fn, Path)
assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz')
def test_load_cached_data_for_updating(mocker) -> None:
datadir = Path(__file__).parent.parent.joinpath('testdata')
@ -569,3 +586,92 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
assert "ETH/BTC" in unav_pairs
assert "XRP/BTC" in unav_pairs
assert log_has("Skipping pair ETH/BTC...", caplog)
def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102")
unavailable_pairs = refresh_backtest_trades_data(exchange=ex,
pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"],
datadir=testdatadir,
timerange=timerange, erase=True
)
assert dl_mock.call_count == 2
assert dl_mock.call_args[1]['timerange'].starttype == 'date'
assert log_has("Downloading trades for pair ETH/BTC.", caplog)
assert unavailable_pairs == ["XRP/ETH"]
assert log_has("Skipping pair XRP/ETH...", caplog)
def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog) -> None:
ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history))
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
ght_mock)
exchange = get_patched_exchange(mocker, default_conf)
file1 = testdatadir / 'ETH_BTC-trades.json.gz'
_backup_file(file1)
assert not file1.is_file()
assert download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert log_has("New Amount of trades: 5", caplog)
assert file1.is_file()
# clean files freshly downloaded
_clean_test_file(file1)
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
MagicMock(side_effect=ValueError))
assert not download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog):
pair = 'XRP/ETH'
file1 = testdatadir / 'XRP_ETH-1m.json'
file5 = testdatadir / 'XRP_ETH-5m.json'
# Compare downloaded dataset with converted dataset
dfbak_1m = history.load_pair_history(datadir=testdatadir,
ticker_interval="1m",
pair=pair)
dfbak_5m = history.load_pair_history(datadir=testdatadir,
ticker_interval="5m",
pair=pair)
_backup_file(file1, copy_file=True)
_backup_file(file5)
tr = TimeRange.parse_timerange('20191011-20191012')
convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
datadir=testdatadir, timerange=tr, erase=True)
assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
# Load new data
df_1m = history.load_pair_history(datadir=testdatadir,
ticker_interval="1m",
pair=pair)
df_5m = history.load_pair_history(datadir=testdatadir,
ticker_interval="5m",
pair=pair)
assert df_1m.equals(dfbak_1m)
assert df_5m.equals(dfbak_5m)
_clean_test_file(file1)
_clean_test_file(file5)

View File

@ -1142,6 +1142,13 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_
await exchange._async_get_candle_history(pair, "5m",
(arrow.utcnow().timestamp - 2000) * 1000)
with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
r'historical candlestick data\..*'):
api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
await exchange._async_get_candle_history(pair, "5m",
(arrow.utcnow().timestamp - 2000) * 1000)
@pytest.mark.asyncio
async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
@ -1313,6 +1320,196 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na
assert ticks[9][5] == 2.31452783
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
trades_history):
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
# Monkey-patch async function
exchange._api_async.fetch_trades = get_mock_coro(trades_history)
pair = 'ETH/BTC'
res = await exchange._async_fetch_trades(pair, since=None, params=None)
assert type(res) is list
assert isinstance(res[0], dict)
assert isinstance(res[1], dict)
assert exchange._api_async.fetch_trades.call_count == 1
assert exchange._api_async.fetch_trades.call_args[0][0] == pair
assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
assert log_has_re(f"Fetching trades for pair {pair}, since .*", caplog)
caplog.clear()
exchange._api_async.fetch_trades.reset_mock()
res = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'})
assert exchange._api_async.fetch_trades.call_count == 1
assert exchange._api_async.fetch_trades.call_args[0][0] == pair
assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
assert exchange._api_async.fetch_trades.call_args[1]['params'] == {'from': '123'}
assert log_has_re(f"Fetching trades for pair {pair}, params: .*", caplog)
exchange = Exchange(default_conf)
await async_ccxt_exception(mocker, default_conf, MagicMock(),
"_async_fetch_trades", "fetch_trades",
pair='ABCD/BTC', since=None)
api_mock = MagicMock()
with pytest.raises(OperationalException, match=r'Could not fetch trade data*'):
api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
r'historical trade data\..*'):
api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchange_name,
trades_history):
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
pagination_arg = exchange._trades_pagination_arg
async def mock_get_trade_hist(pair, *args, **kwargs):
if 'since' in kwargs:
# Return first 3
return trades_history[:-2]
elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']:
# Return 2
return trades_history[-3:-1]
else:
# Return last 2
return trades_history[-2:]
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
assert len(ret[1]) == len(trades_history)
assert exchange._async_fetch_trades.call_count == 3
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
# 2nd call
assert fetch_trades_cal[1][0][0] == pair
assert 'params' in fetch_trades_cal[1][1]
assert exchange._ft_has['trades_pagination_arg'] in fetch_trades_cal[1][1]['params']
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_get_trade_history_time(default_conf, mocker, caplog, exchange_name,
trades_history):
caplog.set_level(logging.DEBUG)
async def mock_get_trade_hist(pair, *args, **kwargs):
if kwargs['since'] == trades_history[0]["timestamp"]:
return trades_history[:-1]
else:
return trades_history[-1:]
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
assert len(ret[1]) == len(trades_history)
assert exchange._async_fetch_trades.call_count == 2
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
# 2nd call
assert fetch_trades_cal[1][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
assert log_has_re(r"Stopping because until was reached.*", caplog)
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, exchange_name,
trades_history):
caplog.set_level(logging.DEBUG)
async def mock_get_trade_hist(pair, *args, **kwargs):
if kwargs['since'] == trades_history[0]["timestamp"]:
return trades_history[:-1]
else:
return []
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
# Monkey-patch async function
exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
pair = 'ETH/BTC'
ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"]-1)
assert type(ret) is tuple
assert ret[0] == pair
assert type(ret[1]) is list
assert len(ret[1]) == len(trades_history) - 1
assert exchange._async_fetch_trades.call_count == 2
fetch_trades_cal = exchange._async_fetch_trades.call_args_list
# first call (using since, not fromId)
assert fetch_trades_cal[0][0][0] == pair
assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades_history):
mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
pair = 'ETH/BTC'
exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history))
exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history))
ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"])
# Depending on the exchange, one or the other method should be called
assert sum([exchange._async_get_trade_history_id.call_count,
exchange._async_get_trade_history_time.call_count]) == 1
assert len(ret) == 2
assert ret[0] == pair
assert len(ret[1]) == len(trades_history)
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange_name,
trades_history):
mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=False)
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
pair = 'ETH/BTC'
with pytest.raises(OperationalException,
match="This exchange does not suport downloading Trades."):
exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
until=trades_history[-1]["timestamp"])
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
default_conf['dry_run'] = True
@ -1459,13 +1656,17 @@ def test_merge_ft_has_dict(default_conf, mocker):
assert ex._ft_has == Exchange._ft_has_default
ex = Kraken(default_conf)
assert ex._ft_has == Exchange._ft_has_default
assert ex._ft_has != Exchange._ft_has_default
assert ex._ft_has['trades_pagination'] == 'id'
assert ex._ft_has['trades_pagination_arg'] == 'since'
# Binance defines different values
ex = Binance(default_conf)
assert ex._ft_has != Exchange._ft_has_default
assert ex._ft_has['stoploss_on_exchange']
assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
assert ex._ft_has['trades_pagination'] == 'id'
assert ex._ft_has['trades_pagination_arg'] == 'fromId'
conf = copy.deepcopy(default_conf)
conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,

View File

@ -260,3 +260,25 @@ def test_download_data_no_pairs(mocker, caplog):
with pytest.raises(OperationalException,
match=r"Downloading data requires a list of pairs\..*"):
start_download_data(pargs)
def test_download_data_trades(mocker, caplog):
dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data',
MagicMock(return_value=[]))
convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv',
MagicMock(return_value=[]))
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
args = [
"download-data",
"--exchange", "kraken",
"--pairs", "ETH/BTC", "XRP/BTC",
"--days", "20",
"--dl-trades"
]
start_download_data(get_args(args))
assert dl_mock.call_args[1]['timerange'].starttype == "date"
assert dl_mock.call_count == 1
assert convert_mock.call_count == 1

1
tests/testdata/XRP_ETH-1m.json vendored Normal file

File diff suppressed because one or more lines are too long

1
tests/testdata/XRP_ETH-5m.json vendored Normal file

File diff suppressed because one or more lines are too long

BIN
tests/testdata/XRP_ETH-trades.json.gz vendored Normal file

Binary file not shown.