adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use handler for trades

Browse Source
This commit is contained in:
Matthias 2019-12-25 16:34:27 +01:00
parent 9547d47ae2
commit 9876d126ca
3 changed files with 26 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
freqtrade/data/converter.py
View File

@ -155,12 +155,12 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
return frame return frame
def trades_to_ohlcv(trades: list, timeframe: str) -> list: def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
""" """
Converts trades list to ohlcv list Converts trades list to ohlcv list
:param trades: List of trades, as returned by ccxt.fetch_trades. :param trades: List of trades, as returned by ccxt.fetch_trades.
:param timeframe: Ticker timeframe to resample data to :param timeframe: Ticker timeframe to resample data to
:return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv) :return: ohlcv Dataframe.
""" """
from freqtrade.exchange import timeframe_to_minutes from freqtrade.exchange import timeframe_to_minutes
ticker_minutes = timeframe_to_minutes(timeframe) ticker_minutes = timeframe_to_minutes(timeframe)
@ -170,8 +170,7 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> list:
df_new = df['price'].resample(f'{ticker_minutes}min').ohlc() df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum() df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
df_new['date'] = df_new.index.astype("int64") // 10 ** 6 df_new['date'] = df_new.index
# Drop 0 volume rows # Drop 0 volume rows
df_new = df_new.dropna() df_new = df_new.dropna()
columns = ["date", "open", "high", "low", "close", "volume"] return df_new[['date', 'open', 'high', 'low', 'close', 'volume']]
return list(zip(*[df_new[x].values.tolist() for x in columns]))

38
freqtrade/data/history.py
View File

@ -18,7 +18,7 @@ from pandas import DataFrame
from freqtrade import OperationalException, misc from freqtrade import OperationalException, misc
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data.converter import trades_to_ohlcv from freqtrade.data.converter import trades_to_ohlcv
from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass from freqtrade.data.datahandlers import get_datahandler
from freqtrade.data.datahandlers.idatahandler import IDataHandler from freqtrade.data.datahandlers.idatahandler import IDataHandler
from freqtrade.exchange import Exchange, timeframe_to_minutes from freqtrade.exchange import Exchange, timeframe_to_minutes
@ -90,15 +90,6 @@ def load_trades_file(datadir: Path, pair: str,
return tradesdata return tradesdata
def store_trades_file(datadir: Path, pair: str,
data: list, is_zip: bool = True):
"""
Stores tickerdata to file
"""
filename = pair_trades_filename(datadir, pair)
misc.file_dump_json(filename, data, is_zip=is_zip)
def load_pair_history(pair: str, def load_pair_history(pair: str,
timeframe: str, timeframe: str,
datadir: Path, *, datadir: Path, *,
@ -339,10 +330,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
return pairs_not_available return pairs_not_available
def _download_trades_history(datadir: Path, def _download_trades_history(exchange: Exchange,
exchange: Exchange, pair: str, *,
pair: str, timerange: Optional[TimeRange] = None,
timerange: Optional[TimeRange] = None) -> bool: data_handler: IDataHandler
) -> bool:
""" """
Download trade history from the exchange. Download trade history from the exchange.
Appends to previously downloaded trades data. Appends to previously downloaded trades data.
@ -351,7 +343,7 @@ def _download_trades_history(datadir: Path,
since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
trades = load_trades_file(datadir, pair) trades = data_handler.trades_load(pair)
from_id = trades[-1]['id'] if trades else None from_id = trades[-1]['id'] if trades else None
@ -366,7 +358,7 @@ def _download_trades_history(datadir: Path,
from_id=from_id, from_id=from_id,
) )
trades.extend(new_trades[1]) trades.extend(new_trades[1])
store_trades_file(datadir, pair, trades) data_handler.trades_store(pair, data=trades)
logger.debug("New Start: %s", trades[0]['datetime']) logger.debug("New Start: %s", trades[0]['datetime'])
logger.debug("New End: %s", trades[-1]['datetime']) logger.debug("New End: %s", trades[-1]['datetime'])
@ -382,13 +374,15 @@ def _download_trades_history(datadir: Path,
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, erase=False) -> List[str]: timerange: TimeRange, erase=False,
data_format: str = 'jsongz') -> List[str]:
""" """
Refresh stored trades data for backtesting and hyperopt operations. Refresh stored trades data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand. Used by freqtrade download-data subcommand.
:return: List of pairs that are not available. :return: List of pairs that are not available.
""" """
pairs_not_available = [] pairs_not_available = []
data_handler = get_datahandler(datadir, data_format=data_format)
for pair in pairs: for pair in pairs:
if pair not in exchange.markets: if pair not in exchange.markets:
pairs_not_available.append(pair) pairs_not_available.append(pair)
@ -404,7 +398,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
logger.info(f'Downloading trades for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.')
_download_trades_history(datadir=datadir, exchange=exchange, _download_trades_history(datadir=datadir, exchange=exchange,
pair=pair, pair=pair,
timerange=timerange) timerange=timerange,
data_handler=data_handler)
return pairs_not_available return pairs_not_available
@ -413,8 +408,11 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
""" """
Convert stored trades data to ohlcv data Convert stored trades data to ohlcv data
""" """
data_handler_trades = get_datahandler(datadir, data_format='jsongz')
data_handler_ohlcv = get_datahandler(datadir, data_format='json')
for pair in pairs: for pair in pairs:
trades = load_trades_file(datadir, pair) trades = data_handler_trades.trades_load(pair)
for timeframe in timeframes: for timeframe in timeframes:
ohlcv_file = pair_data_filename(datadir, pair, timeframe) ohlcv_file = pair_data_filename(datadir, pair, timeframe)
if erase and ohlcv_file.exists(): if erase and ohlcv_file.exists():
@ -422,7 +420,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
ohlcv_file.unlink() ohlcv_file.unlink()
ohlcv = trades_to_ohlcv(trades, timeframe) ohlcv = trades_to_ohlcv(trades, timeframe)
# Store ohlcv # Store ohlcv
store_tickerdata_file(datadir, pair, timeframe, data=ohlcv) data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv)
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:

7
tests/data/test_history.py
View File

@ -10,6 +10,7 @@ import arrow
from pandas import DataFrame from pandas import DataFrame
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data.datahandlers import get_datahandler
from freqtrade.data.history import (_download_pair_history, from freqtrade.data.history import (_download_pair_history,
_download_trades_history, _download_trades_history,
_load_cached_data_for_updating, _load_cached_data_for_updating,
@ -597,12 +598,12 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
ght_mock) ght_mock)
exchange = get_patched_exchange(mocker, default_conf) exchange = get_patched_exchange(mocker, default_conf)
file1 = testdatadir / 'ETH_BTC-trades.json.gz' file1 = testdatadir / 'ETH_BTC-trades.json.gz'
data_handler = get_datahandler(testdatadir, data_format='jsongz')
_backup_file(file1) _backup_file(file1)
assert not file1.is_file() assert not file1.is_file()
assert _download_trades_history(datadir=testdatadir, exchange=exchange, assert _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='ETH/BTC') pair='ETH/BTC')
assert log_has("New Amount of trades: 5", caplog) assert log_has("New Amount of trades: 5", caplog)
assert file1.is_file() assert file1.is_file()
@ -613,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
MagicMock(side_effect=ValueError)) MagicMock(side_effect=ValueError))
assert not _download_trades_history(datadir=testdatadir, exchange=exchange, assert not _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='ETH/BTC') pair='ETH/BTC')
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)