adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Minor improvements in data.history

Browse Source
This commit is contained in:
hroff-1902 2019-12-18 01:06:03 +03:00
parent 021fa1ca1a
commit cf4c3642ce
9 changed files with 44 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
freqtrade/data/history.py
View File

@ -68,7 +68,7 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date
def load_tickerdata_file(datadir: Path, pair: str, timeframe: str,
timerange: Optional[TimeRange] = None) -> Optional[list]:
timerange: Optional[TimeRange] = None) -> List[Dict]:
"""
Load a pair from file, either .json.gz or .json
:return: tickerlist or None if unsuccessful
@ -276,7 +276,7 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str,
def _download_pair_history(datadir: Path,
exchange: Optional[Exchange],
exchange: Exchange,
pair: str,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None) -> bool:
@ -293,11 +293,6 @@ def _download_pair_history(datadir: Path,
:param timerange: range of time to download
:return: bool with success state
"""
if not exchange:
raise OperationalException(
"Exchange needs to be initialized when downloading pair history data"
)
try:
logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
@ -447,18 +442,19 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
store_tickerdata_file(datadir, pair, timeframe, data=ohlcv)
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
"""
Get the maximum timeframe for the given backtest data
Get the maximum common timerange for the given backtest data.
:param data: dictionary with preprocessed backtesting data
:return: tuple containing min_date, max_date
"""
timeframe = [
timeranges = [
(arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
for frame in data.values()
]
return min(timeframe, key=operator.itemgetter(0))[0], \
max(timeframe, key=operator.itemgetter(1))[1]
return (min(timeranges, key=operator.itemgetter(0))[0],
max(timeranges, key=operator.itemgetter(1))[1])
def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime,

2
freqtrade/edge/__init__.py
View File

@ -120,7 +120,7 @@ class Edge:
preprocessed = self.strategy.tickerdata_to_dataframe(data)
# Print timeframe
min_date, max_date = history.get_timeframe(preprocessed)
min_date, max_date = history.get_timerange(preprocessed)
logger.info(
'Measuring data from %s up to %s (%s days) ...',
min_date.isoformat(),

4
freqtrade/optimize/backtesting.py
View File

@ -117,7 +117,7 @@ class Backtesting:
fail_without_data=True,
)
min_date, max_date = history.get_timeframe(data)
min_date, max_date = history.get_timerange(data)
logger.info(
'Loading data from %s up to %s (%s days)..',
@ -481,7 +481,7 @@ class Backtesting:
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = history.trim_dataframe(df, timerange)
min_date, max_date = history.get_timeframe(preprocessed)
min_date, max_date = history.get_timerange(preprocessed)
logger.info(
'Backtesting with data from %s up to %s (%s days)..',

6
freqtrade/optimize/hyperopt.py
View File

@ -23,7 +23,7 @@ from joblib import (Parallel, cpu_count, delayed, dump, load,
from pandas import DataFrame
from freqtrade import OperationalException
from freqtrade.data.history import get_timeframe, trim_dataframe
from freqtrade.data.history import get_timerange, trim_dataframe
from freqtrade.misc import plural, round_dict
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
@ -369,7 +369,7 @@ class Hyperopt:
processed = load(self.tickerdata_pickle)
min_date, max_date = get_timeframe(processed)
min_date, max_date = get_timerange(processed)
backtesting_results = self.backtesting.backtest(
{
@ -490,7 +490,7 @@ class Hyperopt:
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = trim_dataframe(df, timerange)
min_date, max_date = get_timeframe(data)
min_date, max_date = get_timerange(data)
logger.info(
'Hyperopting with data from %s up to %s (%s days)..',

4
tests/data/test_converter.py
View File

@ -2,7 +2,7 @@
import logging
from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data
from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timeframe
from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timerange
from tests.conftest import log_has
@ -36,7 +36,7 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
f"{len(data)} - after: {len(data2)}", caplog)
# Test fillup actually fixes invalid backtest data
min_date, max_date = get_timeframe({'UNITTEST/BTC': data})
min_date, max_date = get_timerange({'UNITTEST/BTC': data})
assert validate_backtest_data(data, 'UNITTEST/BTC', min_date, max_date, 1)
assert not validate_backtest_data(data2, 'UNITTEST/BTC', min_date, max_date, 1)

15
tests/data/test_history.py
View File

@ -7,15 +7,13 @@ from shutil import copyfile
from unittest.mock import MagicMock, PropertyMock
import arrow
import pytest
from pandas import DataFrame
from freqtrade import OperationalException
from freqtrade.configuration import TimeRange
from freqtrade.data.history import (_download_pair_history,
_download_trades_history,
_load_cached_data_for_updating,
convert_trades_to_ohlcv, get_timeframe,
convert_trades_to_ohlcv, get_timerange,
load_data, load_pair_history,
load_tickerdata_file, pair_data_filename,
pair_trades_filename,
@ -138,9 +136,6 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
'Download history data for pair: "MEME/BTC", timeframe: 1m '
'and store in .*', caplog
)
with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'):
refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'],
exchange=None)
_clean_test_file(file)
@ -512,7 +507,7 @@ def test_file_dump_json_tofile(testdatadir) -> None:
_clean_test_file(file)
def test_get_timeframe(default_conf, mocker, testdatadir) -> None:
def test_get_timerange(default_conf, mocker, testdatadir) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
@ -523,7 +518,7 @@ def test_get_timeframe(default_conf, mocker, testdatadir) -> None:
pairs=['UNITTEST/BTC']
)
)
min_date, max_date = get_timeframe(data)
min_date, max_date = get_timerange(data)
assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
assert max_date.isoformat() == '2017-11-14T22:58:00+00:00'
@ -540,7 +535,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir)
fill_up_missing=False
)
)
min_date, max_date = get_timeframe(data)
min_date, max_date = get_timerange(data)
caplog.clear()
assert validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
min_date, max_date, timeframe_to_minutes('1m'))
@ -564,7 +559,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No
)
)
min_date, max_date = get_timeframe(data)
min_date, max_date = get_timerange(data)
caplog.clear()
assert not validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
min_date, max_date, timeframe_to_minutes('5m'))

4
tests/optimize/test_backtest_detail.py
View File

@ -4,7 +4,7 @@ from unittest.mock import MagicMock
import pytest
from freqtrade.data.history import get_timeframe
from freqtrade.data.history import get_timerange
from freqtrade.optimize.backtesting import Backtesting
from freqtrade.strategy.interface import SellType
from tests.conftest import patch_exchange
@ -380,7 +380,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
pair = "UNITTEST/BTC"
# Dummy data as we mock the analyze functions
data_processed = {pair: frame.copy()}
min_date, max_date = get_timeframe({pair: frame})
min_date, max_date = get_timerange({pair: frame})
results = backtesting.backtest(
{
'stake_amount': default_conf['stake_amount'],

20
tests/optimize/test_backtesting.py
View File

@ -16,7 +16,7 @@ from freqtrade.data import history
from freqtrade.data.btanalysis import evaluate_result_multi
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import get_timeframe
from freqtrade.data.history import get_timerange
from freqtrade.optimize import setup_configuration, start_backtesting
from freqtrade.optimize.backtesting import Backtesting
from freqtrade.state import RunMode
@ -100,7 +100,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
data = load_data_test(contour, testdatadir)
processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timeframe(processed)
min_date, max_date = get_timerange(processed)
assert isinstance(processed, dict)
results = backtesting.backtest(
{
@ -138,7 +138,7 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=
patch_exchange(mocker)
backtesting = Backtesting(conf)
processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timeframe(processed)
min_date, max_date = get_timerange(processed)
return {
'stake_amount': conf['stake_amount'],
'processed': processed,
@ -458,11 +458,11 @@ def test_generate_text_table_strategyn(default_conf, mocker):
def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
def get_timeframe(input1):
def get_timerange(input1):
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe)
mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
patch_exchange(mocker)
mocker.patch.multiple(
@ -491,11 +491,11 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None:
def get_timeframe(input1):
def get_timerange(input1):
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame()))
mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe)
mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
patch_exchange(mocker)
mocker.patch.multiple(
@ -525,7 +525,7 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'],
timerange=timerange)
data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timeframe(data_processed)
min_date, max_date = get_timerange(data_processed)
results = backtesting.backtest(
{
'stake_amount': default_conf['stake_amount'],
@ -581,7 +581,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) -
data = history.load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'],
timerange=timerange)
processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timeframe(processed)
min_date, max_date = get_timerange(processed)
results = backtesting.backtest(
{
'stake_amount': default_conf['stake_amount'],
@ -701,7 +701,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
backtesting.strategy.advise_sell = _trend_alternate_hold # Override
data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timeframe(data_processed)
min_date, max_date = get_timerange(data_processed)
backtest_conf = {
'stake_amount': default_conf['stake_amount'],
'processed': data_processed,

22
tests/optimize/test_hyperopt.py
View File

@ -251,7 +251,7 @@ def test_start_no_data(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -427,7 +427,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -602,7 +602,7 @@ def test_generate_optimizer(mocker, default_conf) -> None:
MagicMock(return_value=backtest_result)
)
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13)))
)
patch_exchange(mocker)
@ -726,7 +726,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -769,7 +769,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -811,7 +811,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -851,7 +851,7 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys)
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -899,7 +899,7 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -930,7 +930,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -977,7 +977,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
@ -1030,7 +1030,7 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
MagicMock(return_value=(MagicMock(), None)))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
'freqtrade.optimize.hyperopt.get_timerange',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)