Merge pull request #2683 from hroff-1902/minor-data-history-4

Minor improvements in data.history

freqtrade/data/history.py

@@ -68,7 +68,7 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date
 
 
 def load_tickerdata_file(datadir: Path, pair: str, timeframe: str,
-                         timerange: Optional[TimeRange] = None) -> Optional[list]:
+                         timerange: Optional[TimeRange] = None) -> List[Dict]:
     """
     Load a pair from file, either .json.gz or .json
     :return: tickerlist or None if unsuccessful
@@ -276,7 +276,7 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str,
 
 
 def _download_pair_history(datadir: Path,
-                           exchange: Optional[Exchange],
+                           exchange: Exchange,
                            pair: str,
                            timeframe: str = '5m',
                            timerange: Optional[TimeRange] = None) -> bool:
@@ -293,11 +293,6 @@ def _download_pair_history(datadir: Path,
     :param timerange: range of time to download
     :return: bool with success state
     """
-    if not exchange:
-        raise OperationalException(
-            "Exchange needs to be initialized when downloading pair history data"
-        )
-
     try:
         logger.info(
             f'Download history data for pair: "{pair}", timeframe: {timeframe} '
@@ -447,18 +442,19 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
             store_tickerdata_file(datadir, pair, timeframe, data=ohlcv)
 
 
-def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
+def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
     """
-    Get the maximum timeframe for the given backtest data
+    Get the maximum common timerange for the given backtest data.
+
     :param data: dictionary with preprocessed backtesting data
     :return: tuple containing min_date, max_date
     """
-    timeframe = [
+    timeranges = [
         (arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
         for frame in data.values()
     ]
-    return min(timeframe, key=operator.itemgetter(0))[0], \
+    return (min(timeranges, key=operator.itemgetter(0))[0],
-        max(timeframe, key=operator.itemgetter(1))[1]
+            max(timeranges, key=operator.itemgetter(1))[1])
 
 
 def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime,

freqtrade/edge/__init__.py

@@ -120,7 +120,7 @@ class Edge:
         preprocessed = self.strategy.tickerdata_to_dataframe(data)
 
         # Print timeframe
-        min_date, max_date = history.get_timeframe(preprocessed)
+        min_date, max_date = history.get_timerange(preprocessed)
         logger.info(
             'Measuring data from %s up to %s (%s days) ...',
             min_date.isoformat(),

freqtrade/optimize/backtesting.py

@@ -117,7 +117,7 @@ class Backtesting:
             fail_without_data=True,
         )
 
-        min_date, max_date = history.get_timeframe(data)
+        min_date, max_date = history.get_timerange(data)
 
         logger.info(
             'Loading data from %s up to %s (%s days)..',
@@ -481,7 +481,7 @@ class Backtesting:
             # Trim startup period from analyzed dataframe
             for pair, df in preprocessed.items():
                 preprocessed[pair] = history.trim_dataframe(df, timerange)
-            min_date, max_date = history.get_timeframe(preprocessed)
+            min_date, max_date = history.get_timerange(preprocessed)
 
             logger.info(
                 'Backtesting with data from %s up to %s (%s days)..',

freqtrade/optimize/hyperopt.py

@@ -23,7 +23,7 @@ from joblib import (Parallel, cpu_count, delayed, dump, load,
 from pandas import DataFrame
 
 from freqtrade import OperationalException
-from freqtrade.data.history import get_timeframe, trim_dataframe
+from freqtrade.data.history import get_timerange, trim_dataframe
 from freqtrade.misc import plural, round_dict
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
@@ -369,7 +369,7 @@ class Hyperopt:
 
         processed = load(self.tickerdata_pickle)
 
-        min_date, max_date = get_timeframe(processed)
+        min_date, max_date = get_timerange(processed)
 
         backtesting_results = self.backtesting.backtest(
             {
@@ -490,7 +490,7 @@ class Hyperopt:
         # Trim startup period from analyzed dataframe
         for pair, df in preprocessed.items():
             preprocessed[pair] = trim_dataframe(df, timerange)
-        min_date, max_date = get_timeframe(data)
+        min_date, max_date = get_timerange(data)
 
         logger.info(
             'Hyperopting with data from %s up to %s (%s days)..',

tests/data/test_converter.py

@@ -2,7 +2,7 @@
 import logging
 
 from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data
-from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timeframe
+from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timerange
 from tests.conftest import log_has
 
 
@@ -36,7 +36,7 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
                    f"{len(data)} - after: {len(data2)}", caplog)
 
     # Test fillup actually fixes invalid backtest data
-    min_date, max_date = get_timeframe({'UNITTEST/BTC': data})
+    min_date, max_date = get_timerange({'UNITTEST/BTC': data})
     assert validate_backtest_data(data, 'UNITTEST/BTC', min_date, max_date, 1)
     assert not validate_backtest_data(data2, 'UNITTEST/BTC', min_date, max_date, 1)
 

tests/data/test_history.py

@@ -7,15 +7,13 @@ from shutil import copyfile
 from unittest.mock import MagicMock, PropertyMock
 
 import arrow
-import pytest
 from pandas import DataFrame
 
-from freqtrade import OperationalException
 from freqtrade.configuration import TimeRange
 from freqtrade.data.history import (_download_pair_history,
                                     _download_trades_history,
                                     _load_cached_data_for_updating,
-                                    convert_trades_to_ohlcv, get_timeframe,
+                                    convert_trades_to_ohlcv, get_timerange,
                                     load_data, load_pair_history,
                                     load_tickerdata_file, pair_data_filename,
                                     pair_trades_filename,
@@ -138,9 +136,6 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
         'Download history data for pair: "MEME/BTC", timeframe: 1m '
         'and store in .*', caplog
     )
-    with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'):
-        refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'],
-                     exchange=None)
     _clean_test_file(file)
 
 
@@ -512,7 +507,7 @@ def test_file_dump_json_tofile(testdatadir) -> None:
     _clean_test_file(file)
 
 
-def test_get_timeframe(default_conf, mocker, testdatadir) -> None:
+def test_get_timerange(default_conf, mocker, testdatadir) -> None:
     patch_exchange(mocker)
     strategy = DefaultStrategy(default_conf)
 
@@ -523,7 +518,7 @@ def test_get_timeframe(default_conf, mocker, testdatadir) -> None:
             pairs=['UNITTEST/BTC']
         )
     )
-    min_date, max_date = get_timeframe(data)
+    min_date, max_date = get_timerange(data)
     assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
     assert max_date.isoformat() == '2017-11-14T22:58:00+00:00'
 
@@ -540,7 +535,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir)
             fill_up_missing=False
         )
     )
-    min_date, max_date = get_timeframe(data)
+    min_date, max_date = get_timerange(data)
     caplog.clear()
     assert validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
                                   min_date, max_date, timeframe_to_minutes('1m'))
@@ -564,7 +559,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No
         )
     )
 
-    min_date, max_date = get_timeframe(data)
+    min_date, max_date = get_timerange(data)
     caplog.clear()
     assert not validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
                                       min_date, max_date, timeframe_to_minutes('5m'))

tests/optimize/test_backtest_detail.py

@@ -4,7 +4,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.data.history import get_timeframe
+from freqtrade.data.history import get_timerange
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.strategy.interface import SellType
 from tests.conftest import patch_exchange
@@ -380,7 +380,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     pair = "UNITTEST/BTC"
     # Dummy data as we mock the analyze functions
     data_processed = {pair: frame.copy()}
-    min_date, max_date = get_timeframe({pair: frame})
+    min_date, max_date = get_timerange({pair: frame})
     results = backtesting.backtest(
         {
             'stake_amount': default_conf['stake_amount'],

tests/optimize/test_backtesting.py

@@ -16,7 +16,7 @@ from freqtrade.data import history
 from freqtrade.data.btanalysis import evaluate_result_multi
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.data.history import get_timeframe
+from freqtrade.data.history import get_timerange
 from freqtrade.optimize import setup_configuration, start_backtesting
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.state import RunMode
@@ -100,7 +100,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
 
     data = load_data_test(contour, testdatadir)
     processed = backtesting.strategy.tickerdata_to_dataframe(data)
-    min_date, max_date = get_timeframe(processed)
+    min_date, max_date = get_timerange(processed)
     assert isinstance(processed, dict)
     results = backtesting.backtest(
         {
@@ -138,7 +138,7 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=
     patch_exchange(mocker)
     backtesting = Backtesting(conf)
     processed = backtesting.strategy.tickerdata_to_dataframe(data)
-    min_date, max_date = get_timeframe(processed)
+    min_date, max_date = get_timerange(processed)
     return {
         'stake_amount': conf['stake_amount'],
         'processed': processed,
@@ -458,11 +458,11 @@ def test_generate_text_table_strategyn(default_conf, mocker):
 
 
 def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
-    def get_timeframe(input1):
+    def get_timerange(input1):
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
     mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
-    mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe)
+    mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -491,11 +491,11 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
 
 
 def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None:
-    def get_timeframe(input1):
+    def get_timerange(input1):
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
     mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame()))
-    mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe)
+    mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -525,7 +525,7 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
     data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'],
                              timerange=timerange)
     data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
-    min_date, max_date = get_timeframe(data_processed)
+    min_date, max_date = get_timerange(data_processed)
     results = backtesting.backtest(
         {
             'stake_amount': default_conf['stake_amount'],
@@ -581,7 +581,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) -
     data = history.load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'],
                              timerange=timerange)
     processed = backtesting.strategy.tickerdata_to_dataframe(data)
-    min_date, max_date = get_timeframe(processed)
+    min_date, max_date = get_timerange(processed)
     results = backtesting.backtest(
         {
             'stake_amount': default_conf['stake_amount'],
@@ -701,7 +701,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
     backtesting.strategy.advise_sell = _trend_alternate_hold  # Override
 
     data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
-    min_date, max_date = get_timeframe(data_processed)
+    min_date, max_date = get_timerange(data_processed)
     backtest_conf = {
         'stake_amount': default_conf['stake_amount'],
         'processed': data_processed,

tests/optimize/test_hyperopt.py

@@ -251,7 +251,7 @@ def test_start_no_data(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -427,7 +427,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -602,7 +602,7 @@ def test_generate_optimizer(mocker, default_conf) -> None:
         MagicMock(return_value=backtest_result)
     )
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13)))
     )
     patch_exchange(mocker)
@@ -726,7 +726,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -769,7 +769,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -811,7 +811,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -851,7 +851,7 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys)
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -899,7 +899,7 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -930,7 +930,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -977,7 +977,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )
 
@@ -1030,7 +1030,7 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
-        'freqtrade.optimize.hyperopt.get_timeframe',
+        'freqtrade.optimize.hyperopt.get_timerange',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
     )