From cf4c3642ce1ea09c758b16a8774cabcea0674ec0 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 18 Dec 2019 01:06:03 +0300 Subject: [PATCH] Minor improvements in data.history --- freqtrade/data/history.py | 20 ++++++++------------ freqtrade/edge/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/optimize/hyperopt.py | 6 +++--- tests/data/test_converter.py | 4 ++-- tests/data/test_history.py | 15 +++++---------- tests/optimize/test_backtest_detail.py | 4 ++-- tests/optimize/test_backtesting.py | 20 ++++++++++---------- tests/optimize/test_hyperopt.py | 22 +++++++++++----------- 9 files changed, 44 insertions(+), 53 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 64f530b53..4c5c0521f 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -68,7 +68,7 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None) -> Optional[list]: + timerange: Optional[TimeRange] = None) -> List[Dict]: """ Load a pair from file, either .json.gz or .json :return: tickerlist or None if unsuccessful @@ -276,7 +276,7 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, def _download_pair_history(datadir: Path, - exchange: Optional[Exchange], + exchange: Exchange, pair: str, timeframe: str = '5m', timerange: Optional[TimeRange] = None) -> bool: @@ -293,11 +293,6 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ - if not exchange: - raise OperationalException( - "Exchange needs to be initialized when downloading pair history data" - ) - try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' @@ -447,18 +442,19 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], store_tickerdata_file(datadir, pair, timeframe, data=ohlcv) -def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: +def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ - Get the maximum timeframe for the given backtest data + Get the maximum common timerange for the given backtest data. + :param data: dictionary with preprocessed backtesting data :return: tuple containing min_date, max_date """ - timeframe = [ + timeranges = [ (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) for frame in data.values() ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] + return (min(timeranges, key=operator.itemgetter(0))[0], + max(timeranges, key=operator.itemgetter(1))[1]) def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 85029905b..e56071a98 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -120,7 +120,7 @@ class Edge: preprocessed = self.strategy.tickerdata_to_dataframe(data) # Print timeframe - min_date, max_date = history.get_timeframe(preprocessed) + min_date, max_date = history.get_timerange(preprocessed) logger.info( 'Measuring data from %s up to %s (%s days) ...', min_date.isoformat(), diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fc60bd310..726257cdd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -117,7 +117,7 @@ class Backtesting: fail_without_data=True, ) - min_date, max_date = history.get_timeframe(data) + min_date, max_date = history.get_timerange(data) logger.info( 'Loading data from %s up to %s (%s days)..', @@ -481,7 +481,7 @@ class Backtesting: # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = history.trim_dataframe(df, timerange) - min_date, max_date = history.get_timeframe(preprocessed) + min_date, max_date = history.get_timerange(preprocessed) logger.info( 'Backtesting with data from %s up to %s (%s days)..', diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cdde2b722..521a4d790 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,7 +23,7 @@ from joblib import (Parallel, cpu_count, delayed, dump, load, from pandas import DataFrame from freqtrade import OperationalException -from freqtrade.data.history import get_timeframe, trim_dataframe +from freqtrade.data.history import get_timerange, trim_dataframe from freqtrade.misc import plural, round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules @@ -369,7 +369,7 @@ class Hyperopt: processed = load(self.tickerdata_pickle) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) backtesting_results = self.backtesting.backtest( { @@ -490,7 +490,7 @@ class Hyperopt: # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = trim_dataframe(df, timerange) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) logger.info( 'Hyperopting with data from %s up to %s (%s days)..', diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 8184167b3..414551c95 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -2,7 +2,7 @@ import logging from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data -from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timeframe +from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timerange from tests.conftest import log_has @@ -36,7 +36,7 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog): f"{len(data)} - after: {len(data2)}", caplog) # Test fillup actually fixes invalid backtest data - min_date, max_date = get_timeframe({'UNITTEST/BTC': data}) + min_date, max_date = get_timerange({'UNITTEST/BTC': data}) assert validate_backtest_data(data, 'UNITTEST/BTC', min_date, max_date, 1) assert not validate_backtest_data(data2, 'UNITTEST/BTC', min_date, max_date, 1) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index c9b198b39..7b3143db9 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -7,15 +7,13 @@ from shutil import copyfile from unittest.mock import MagicMock, PropertyMock import arrow -import pytest from pandas import DataFrame -from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data.history import (_download_pair_history, _download_trades_history, _load_cached_data_for_updating, - convert_trades_to_ohlcv, get_timeframe, + convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, load_tickerdata_file, pair_data_filename, pair_trades_filename, @@ -138,9 +136,6 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) - with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): - refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], - exchange=None) _clean_test_file(file) @@ -512,7 +507,7 @@ def test_file_dump_json_tofile(testdatadir) -> None: _clean_test_file(file) -def test_get_timeframe(default_conf, mocker, testdatadir) -> None: +def test_get_timerange(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) strategy = DefaultStrategy(default_conf) @@ -523,7 +518,7 @@ def test_get_timeframe(default_conf, mocker, testdatadir) -> None: pairs=['UNITTEST/BTC'] ) ) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' @@ -540,7 +535,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) fill_up_missing=False ) ) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) caplog.clear() assert validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('1m')) @@ -564,7 +559,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No ) ) - min_date, max_date = get_timeframe(data) + min_date, max_date = get_timerange(data) caplog.clear() assert not validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('5m')) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index b36c1b9c5..47cb9f353 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest -from freqtrade.data.history import get_timeframe +from freqtrade.data.history import get_timerange from freqtrade.optimize.backtesting import Backtesting from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange @@ -380,7 +380,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: pair = "UNITTEST/BTC" # Dummy data as we mock the analyze functions data_processed = {pair: frame.copy()} - min_date, max_date = get_timeframe({pair: frame}) + min_date, max_date = get_timerange({pair: frame}) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 38a95be7a..0ea35e41f 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.dataprovider import DataProvider -from freqtrade.data.history import get_timeframe +from freqtrade.data.history import get_timerange from freqtrade.optimize import setup_configuration, start_backtesting from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode @@ -100,7 +100,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: data = load_data_test(contour, testdatadir) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) assert isinstance(processed, dict) results = backtesting.backtest( { @@ -138,7 +138,7 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record= patch_exchange(mocker) backtesting = Backtesting(conf) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) return { 'stake_amount': conf['stake_amount'], 'processed': processed, @@ -458,11 +458,11 @@ def test_generate_text_table_strategyn(default_conf, mocker): def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: - def get_timeframe(input1): + def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) - mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( @@ -491,11 +491,11 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: - def get_timeframe(input1): + def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame())) - mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( @@ -525,7 +525,7 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], timerange=timerange) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(data_processed) + min_date, max_date = get_timerange(data_processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -581,7 +581,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) - data = history.load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'], timerange=timerange) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(processed) + min_date, max_date = get_timerange(processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -701,7 +701,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) backtesting.strategy.advise_sell = _trend_alternate_hold # Override data_processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timeframe(data_processed) + min_date, max_date = get_timerange(data_processed) backtest_conf = { 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index c3afa4911..29b8b5b16 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -251,7 +251,7 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame)) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -427,7 +427,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -602,7 +602,7 @@ def test_generate_optimizer(mocker, default_conf) -> None: MagicMock(return_value=backtest_result) ) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13))) ) patch_exchange(mocker) @@ -726,7 +726,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -769,7 +769,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -811,7 +811,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -851,7 +851,7 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -899,7 +899,7 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) - mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -930,7 +930,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -977,7 +977,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) @@ -1030,7 +1030,7 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) )