adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move convert functions to convert module

Browse Source
This commit is contained in:
Matthias 2019-12-28 10:27:49 +01:00
parent 525550e4c7
commit 28787a001c
5 changed files with 73 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/configuration/arguments.py
View File

@ -47,7 +47,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to"]
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",

62
freqtrade/data/converter.py
View File

@ -3,6 +3,7 @@ Functions to convert data from one format to another
"""
import logging
from datetime import datetime, timezone
from typing import Any, Dict
import pandas as pd
from pandas import DataFrame, to_datetime
@ -175,3 +176,64 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
# Drop 0 volume rows
df_new = df_new.dropna()
return df_new[DEFAULT_DATAFRAME_COLUMNS]
def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool):
"""
Convert trades from one format to another format.
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
:param erase: Erase souce data (does not apply if source and target format are identical)
"""
from freqtrade.data.history.idatahandler import get_datahandler
src = get_datahandler(config['datadir'], convert_from)
trg = get_datahandler(config['datadir'], convert_to)
if 'pairs' not in config:
config['pairs'] = src.trades_get_pairs(config['datadir'])
logger.info(f"Converting trades for {config['pairs']}")
for pair in config['pairs']:
data = src.trades_load(pair=pair)
logger.info(f"Converting {len(data)} trades for {pair}")
trg.trades_store(pair, data)
if erase and convert_from != convert_to:
logger.info(f"Deleting source Trade data for {pair}.")
src.trades_purge(pair=pair)
def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool):
"""
Convert ohlcv from one format to another format.
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
:param erase: Erase souce data (does not apply if source and target format are identical)
"""
from freqtrade.data.history.idatahandler import get_datahandler
src = get_datahandler(config['datadir'], convert_from)
trg = get_datahandler(config['datadir'], convert_to)
timeframes = config.get('timeframes', [config.get('ticker_interval')])
logger.info(f"Converting OHLCV for timeframe {timeframes}")
if 'pairs' not in config:
config['pairs'] = []
# Check timeframes or fall back to ticker_interval.
for timeframe in timeframes:
config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'],
timeframe))
logger.info(f"Converting OHLCV for {config['pairs']}")
for timeframe in timeframes:
for pair in config['pairs']:
data = src.ohlcv_load(pair=pair, timeframe=timeframe,
timerange=None,
fill_missing=False,
drop_incomplete=False,
startup_candles=0)
logger.info(f"Converting {len(data)} candles for {pair}")
trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data)
if erase and convert_from != convert_to:
logger.info(f"Deleting source data for {pair} / {timeframe}")
src.ohlcv_purge(pair=pair, timeframe=timeframe)

2
freqtrade/data/history/jsondatahandler.py
View File

@ -62,6 +62,8 @@ class JsonDataHandler(IDataHandler):
:param pair: Pair to load data
:param timeframe: Ticker timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
:return: DataFrame with ohlcv data, or empty DataFrame
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)

58
freqtrade/utils.py
View File

@ -17,8 +17,9 @@ from freqtrade.configuration import (Configuration, TimeRange,
from freqtrade.configuration.directory_operations import (copy_sample_files,
create_userdata_dir)
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
from freqtrade.data.converter import (convert_ohlcv_format,
convert_trades_format)
from freqtrade.data.history import (convert_trades_to_ohlcv,
get_datahandlerclass,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data)
from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
@ -248,55 +249,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
print(tabulate(strats_to_print, headers='keys', tablefmt='pipe'))
def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str):
"""
TODO: move this to converter.py (?)
"""
SrcClass = get_datahandlerclass(convert_from)
TrgClass = get_datahandlerclass(convert_to)
if 'pairs' not in config:
config['pairs'] = SrcClass.trades_get_pairs(config['datadir'])
logger.info(f"Converting trades for {config['pairs']}")
src = SrcClass(config['datadir'])
trg = TrgClass(config['datadir'])
for pair in config['pairs']:
data = src.trades_load(pair=pair)
logger.info(f"Converting {len(data)} trades for {pair}")
trg.trades_store(pair, data)
def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str):
"""
TODO: move this to converter.py (?)
"""
SrcClass = get_datahandlerclass(convert_from)
TrgClass = get_datahandlerclass(convert_to)
timeframes = config.get('timeframes', [config.get('ticker_interval')])
logger.info(f"Converting OHLCV for timeframe {timeframes}")
if 'pairs' not in config:
config['pairs'] = []
# Check timeframes or fall back to ticker_interval.
for timeframe in timeframes:
config['pairs'].extend(SrcClass.ohlcv_get_pairs(config['datadir'],
timeframe))
logger.info(f"Converting OHLCV for {config['pairs']}")
src = SrcClass(config['datadir'])
trg = TrgClass(config['datadir'])
for timeframe in timeframes:
for pair in config['pairs']:
data = src.ohlcv_load(pair=pair, timeframe=timeframe,
timerange=None,
fill_missing=False,
drop_incomplete=False,
startup_candles=0)
logger.info(f"Converting {len(data)} candles for {pair}")
trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data)
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
"""
Convert data from one format to another
@ -304,10 +256,12 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if ohlcv:
convert_ohlcv_format(config,
convert_from=args['format_from'], convert_to=args['format_to'])
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'])
else:
convert_trades_format(config,
convert_from=args['format_from'], convert_to=args['format_to'])
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'])
def start_list_timeframes(args: Dict[str, Any]) -> None:

3
tests/data/test_history.py
View File

@ -463,7 +463,8 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock())
dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history',
MagicMock())
ex = get_patched_exchange(mocker, default_conf)
mocker.patch(