From 9876d126ca0add61c853a99a0a8a531445046dae Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 16:34:27 +0100 Subject: [PATCH] Use handler for trades --- freqtrade/data/converter.py | 9 ++++----- freqtrade/data/history.py | 38 ++++++++++++++++++------------------- tests/data/test_history.py | 7 ++++--- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 52ce3c593..09f7e3278 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -155,12 +155,12 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: return frame -def trades_to_ohlcv(trades: list, timeframe: str) -> list: +def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: """ Converts trades list to ohlcv list :param trades: List of trades, as returned by ccxt.fetch_trades. :param timeframe: Ticker timeframe to resample data to - :return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv) + :return: ohlcv Dataframe. """ from freqtrade.exchange import timeframe_to_minutes ticker_minutes = timeframe_to_minutes(timeframe) @@ -170,8 +170,7 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> list: df_new = df['price'].resample(f'{ticker_minutes}min').ohlc() df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum() - df_new['date'] = df_new.index.astype("int64") // 10 ** 6 + df_new['date'] = df_new.index # Drop 0 volume rows df_new = df_new.dropna() - columns = ["date", "open", "high", "low", "close", "volume"] - return list(zip(*[df_new[x].values.tolist() for x in columns])) + return df_new[['date', 'open', 'high', 'low', 'close', 'volume']] diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4f31ebd60..1970f05d5 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -18,7 +18,7 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import trades_to_ohlcv -from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass +from freqtrade.data.datahandlers import get_datahandler from freqtrade.data.datahandlers.idatahandler import IDataHandler from freqtrade.exchange import Exchange, timeframe_to_minutes @@ -90,15 +90,6 @@ def load_trades_file(datadir: Path, pair: str, return tradesdata -def store_trades_file(datadir: Path, pair: str, - data: list, is_zip: bool = True): - """ - Stores tickerdata to file - """ - filename = pair_trades_filename(datadir, pair) - misc.file_dump_json(filename, data, is_zip=is_zip) - - def load_pair_history(pair: str, timeframe: str, datadir: Path, *, @@ -339,10 +330,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes return pairs_not_available -def _download_trades_history(datadir: Path, - exchange: Exchange, - pair: str, - timerange: Optional[TimeRange] = None) -> bool: +def _download_trades_history(exchange: Exchange, + pair: str, *, + timerange: Optional[TimeRange] = None, + data_handler: IDataHandler + ) -> bool: """ Download trade history from the exchange. Appends to previously downloaded trades data. @@ -351,7 +343,7 @@ def _download_trades_history(datadir: Path, since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None - trades = load_trades_file(datadir, pair) + trades = data_handler.trades_load(pair) from_id = trades[-1]['id'] if trades else None @@ -366,7 +358,7 @@ def _download_trades_history(datadir: Path, from_id=from_id, ) trades.extend(new_trades[1]) - store_trades_file(datadir, pair, trades) + data_handler.trades_store(pair, data=trades) logger.debug("New Start: %s", trades[0]['datetime']) logger.debug("New End: %s", trades[-1]['datetime']) @@ -382,13 +374,15 @@ def _download_trades_history(datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, erase=False) -> List[str]: + timerange: TimeRange, erase=False, + data_format: str = 'jsongz') -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. :return: List of pairs that are not available. """ pairs_not_available = [] + data_handler = get_datahandler(datadir, data_format=data_format) for pair in pairs: if pair not in exchange.markets: pairs_not_available.append(pair) @@ -404,7 +398,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: logger.info(f'Downloading trades for pair {pair}.') _download_trades_history(datadir=datadir, exchange=exchange, pair=pair, - timerange=timerange) + timerange=timerange, + data_handler=data_handler) return pairs_not_available @@ -413,8 +408,11 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], """ Convert stored trades data to ohlcv data """ + data_handler_trades = get_datahandler(datadir, data_format='jsongz') + data_handler_ohlcv = get_datahandler(datadir, data_format='json') + for pair in pairs: - trades = load_trades_file(datadir, pair) + trades = data_handler_trades.trades_load(pair) for timeframe in timeframes: ohlcv_file = pair_data_filename(datadir, pair, timeframe) if erase and ohlcv_file.exists(): @@ -422,7 +420,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], ohlcv_file.unlink() ohlcv = trades_to_ohlcv(trades, timeframe) # Store ohlcv - store_tickerdata_file(datadir, pair, timeframe, data=ohlcv) + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 5f7d0f5a2..77cb6a565 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -10,6 +10,7 @@ import arrow from pandas import DataFrame from freqtrade.configuration import TimeRange +from freqtrade.data.datahandlers import get_datahandler from freqtrade.data.history import (_download_pair_history, _download_trades_history, _load_cached_data_for_updating, @@ -597,12 +598,12 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad ght_mock) exchange = get_patched_exchange(mocker, default_conf) file1 = testdatadir / 'ETH_BTC-trades.json.gz' - + data_handler = get_datahandler(testdatadir, data_format='jsongz') _backup_file(file1) assert not file1.is_file() - assert _download_trades_history(datadir=testdatadir, exchange=exchange, + assert _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') assert log_has("New Amount of trades: 5", caplog) assert file1.is_file() @@ -613,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) - assert not _download_trades_history(datadir=testdatadir, exchange=exchange, + assert not _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)