Merge pull request #637 from arudov/fix/dl-testdata-period2

Time-range download of backtesting data

freqtrade/tests/exchange/test_exchange.py

@@ -324,6 +324,15 @@ def test_get_ticker(default_conf, mocker):
         get_ticker(pair='ETH/BTC', refresh=True)
 
 
+def make_fetch_ohlcv_mock(data):
+    def fetch_ohlcv_mock(pair, timeframe, since):
+        if since:
+            assert since > data[-1][0]
+            return []
+        return data
+    return fetch_ohlcv_mock
+
+
 def test_get_ticker_history(default_conf, mocker):
     api_mock = MagicMock()
     tick = [
@@ -337,7 +346,7 @@ def test_get_ticker_history(default_conf, mocker):
         ]
     ]
     type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True})
-    api_mock.fetch_ohlcv = MagicMock(return_value=tick)
+    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick))
     mocker.patch('freqtrade.exchange._API', api_mock)
 
     # retrieve original ticker
@@ -360,7 +369,7 @@ def test_get_ticker_history(default_conf, mocker):
             10,  # volume (in quote currency)
         ]
     ]
-    api_mock.fetch_ohlcv = MagicMock(return_value=new_tick)
+    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick))
     mocker.patch('freqtrade.exchange._API', api_mock)
 
     ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval'])

freqtrade/tests/optimize/test_optimize.py

@@ -3,12 +3,14 @@
 import json
 import os
 import uuid
+import arrow
 from shutil import copyfile
 
 from freqtrade import optimize
 from freqtrade.misc import file_dump_json
 from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \
-    download_backtesting_testdata, load_tickerdata_file, trim_tickerlist
+    download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \
+    load_cached_data_for_updating
 from freqtrade.tests.conftest import log_has
 
 # Change this if modifying UNITTEST/BTC testdatafile
@@ -145,6 +147,109 @@ def test_download_pairs(ticker_history, mocker) -> None:
     _clean_test_file(file2_5)
 
 
+def test_load_cached_data_for_updating(mocker) -> None:
+    datadir = os.path.join(os.path.dirname(__file__), '..', 'testdata')
+
+    test_data = None
+    test_filename = os.path.join(datadir, 'UNITTEST_BTC-1m.json')
+    with open(test_filename, "rt") as file:
+        test_data = json.load(file)
+
+    # change now time to test 'line' cases
+    # now = last cached item + 1 hour
+    now_ts = test_data[-1][0] / 1000 + 60 * 60
+    mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts))
+
+    # timeframe starts earlier than the cached data
+    # should fully update data
+    timerange = (('date', None), test_data[0][0] / 1000 - 1, None)
+    data, start_ts = load_cached_data_for_updating(test_filename,
+                                                   '1m',
+                                                   timerange)
+    assert data == []
+    assert start_ts == test_data[0][0] - 1000
+
+    # same with 'line' timeframe
+    num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120
+    data, start_ts = load_cached_data_for_updating(test_filename,
+                                                   '1m',
+                                                   ((None, 'line'), None, -num_lines))
+    assert data == []
+    assert start_ts < test_data[0][0] - 1
+
+    # timeframe starts in the center of the cached data
+    # should return the chached data w/o the last item
+    timerange = (('date', None), test_data[0][0] / 1000 + 1, None)
+    data, start_ts = load_cached_data_for_updating(test_filename,
+                                                   '1m',
+                                                   timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # same with 'line' timeframe
+    num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30
+    timerange = ((None, 'line'), None, -num_lines)
+    data, start_ts = load_cached_data_for_updating(test_filename,
+                                                   '1m',
+                                                   timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # timeframe starts after the chached data
+    # should return the chached data w/o the last item
+    timerange = (('date', None), test_data[-1][0] / 1000 + 1, None)
+    data, start_ts = load_cached_data_for_updating(test_filename,
+                                                   '1m',
+                                                   timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # same with 'line' timeframe
+    num_lines = 30
+    timerange = ((None, 'line'), None, -num_lines)
+    data, start_ts = load_cached_data_for_updating(test_filename,
+                                                   '1m',
+                                                   timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # no timeframe is set
+    # should return the chached data w/o the last item
+    num_lines = 30
+    timerange = ((None, 'line'), None, -num_lines)
+    data, start_ts = load_cached_data_for_updating(test_filename,
+                                                   '1m',
+                                                   timerange)
+    assert data == test_data[:-1]
+    assert test_data[-2][0] < start_ts < test_data[-1][0]
+
+    # no datafile exist
+    # should return timestamp start time
+    timerange = (('date', None), now_ts - 10000, None)
+    data, start_ts = load_cached_data_for_updating(test_filename + 'unexist',
+                                                   '1m',
+                                                   timerange)
+    assert data == []
+    assert start_ts == (now_ts - 10000) * 1000
+
+    # same with 'line' timeframe
+    num_lines = 30
+    timerange = ((None, 'line'), None, -num_lines)
+    data, start_ts = load_cached_data_for_updating(test_filename + 'unexist',
+                                                   '1m',
+                                                   timerange)
+    assert data == []
+    assert start_ts == (now_ts - num_lines * 60) * 1000
+
+    # no datafile exist, no timeframe is set
+    # should return an empty array and None
+    data, start_ts = load_cached_data_for_updating(test_filename + 'unexist',
+                                                   '1m',
+                                                   None)
+    assert data == []
+    assert start_ts is None
+
+
 def test_download_pairs_exception(ticker_history, mocker, caplog) -> None:
     mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history)
     mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata',
@@ -168,7 +273,7 @@ def test_download_backtesting_testdata(ticker_history, mocker) -> None:
     # Download a 1 min ticker file
     file1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'XEL_BTC-1m.json')
     _backup_file(file1)
-    download_backtesting_testdata(None, pair="XEL/BTC", interval='1m')
+    download_backtesting_testdata(None, pair="XEL/BTC", tick_interval='1m')
     assert os.path.isfile(file1) is True
     _clean_test_file(file1)
 
@@ -176,7 +281,7 @@ def test_download_backtesting_testdata(ticker_history, mocker) -> None:
     file2 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'STORJ_BTC-5m.json')
     _backup_file(file2)
 
-    download_backtesting_testdata(None, pair="STORJ/BTC", interval='5m')
+    download_backtesting_testdata(None, pair="STORJ/BTC", tick_interval='5m')
     assert os.path.isfile(file2) is True
     _clean_test_file(file2)
 
@@ -188,8 +293,9 @@ def test_download_backtesting_testdata2(mocker) -> None:
     ]
     json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
     mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick)
-    download_backtesting_testdata(None, pair="UNITTEST/BTC", interval='1m')
-    download_backtesting_testdata(None, pair="UNITTEST/BTC", interval='3m')
+
+    download_backtesting_testdata(None, pair="UNITTEST/BTC", tick_interval='1m')
+    download_backtesting_testdata(None, pair="UNITTEST/BTC", tick_interval='3m')
     assert json_dump_mock.call_count == 2
 
 
@@ -222,12 +328,12 @@ def test_trim_tickerlist() -> None:
     ticker_list_len = len(ticker_list)
 
     # Test the pattern ^(-\d+)$
-    # This pattern remove X element from the beginning
-    timerange = ((None, 'line'), None, 5)
+    # This pattern uses the latest N elements
+    timerange = ((None, 'line'), None, -5)
     ticker = trim_tickerlist(ticker_list, timerange)
     ticker_len = len(ticker)
 
-    assert ticker_list_len == ticker_len + 5
+    assert ticker_len == 5
     assert ticker_list[0] is not ticker[0]  # The first element should be different
     assert ticker_list[-1] is ticker[-1]  # The last element must be the same
 
@@ -252,6 +358,37 @@ def test_trim_tickerlist() -> None:
     assert ticker_list[5] is ticker[0]  # The list starts at the index 5
     assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
 
+    # Test the pattern ^(\d{8})-(\d{8})$
+    # This pattern extract a window between the dates
+    timerange = (('date', 'date'), ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is not ticker[0]  # The first element should be different
+    assert ticker_list[5] is ticker[0]  # The list starts at the index 5
+    assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
+
+    # Test the pattern ^-(\d{8})$
+    # This pattern extracts elements from the start to the date
+    timerange = ((None, 'date'), None, ticker_list[10][0] / 1000 - 1)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 10
+    assert ticker_list[0] is ticker[0]  # The start of the list is included
+    assert ticker_list[9] is ticker[-1]  # The element 10 is not included
+
+    # Test the pattern ^(\d{8})-$
+    # This pattern extracts elements from the date to now
+    timerange = (('date', None), ticker_list[10][0] / 1000 - 1, None)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == ticker_list_len - 10
+    assert ticker_list[10] is ticker[0]  # The first element is element #10
+    assert ticker_list[-1] is ticker[-1]  # The last element is the same
+
     # Test a wrong pattern
     # This pattern must return the list unchanged
     timerange = ((None, None), None, 5)

freqtrade/tests/test_arguments.py

@@ -109,6 +109,13 @@ def test_parse_args_dynamic_whitelist_invalid_values() -> None:
 def test_parse_timerange_incorrect() -> None:
     assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200')
     assert (('line', None), 200, None) == Arguments.parse_timerange('200-')
+    assert (('index', 'index'), 200, 500) == Arguments.parse_timerange('200-500')
+
+    assert (('date', None), 1274486400, None) == Arguments.parse_timerange('20100522-')
+    assert ((None, 'date'), None, 1274486400) == Arguments.parse_timerange('-20100522')
+    timerange = Arguments.parse_timerange('20100522-20150730')
+    assert timerange == (('date', 'date'), 1274486400, 1438214400)
+
     with pytest.raises(Exception, match=r'Incorrect syntax.*'):
         Arguments.parse_timerange('-')