Fix review comments. Add support of datetime timeganges

docs/backtesting.md

@@ -80,12 +80,9 @@ The full timerange specification:
 - Use last 123 tickframes of data: `--timerange=-123`
 - Use first 123 tickframes of data: `--timerange=123-`
 - Use tickframes from line 123 through 456: `--timerange=123-456`
-
+- Use tickframes till 2018/01/31: `--timerange=-20180131`
-
+- Use tickframes since 2018/01/31: `--timerange=20180131-`
-Incoming feature, not implemented yet:
+- Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301`
-- `--timerange=-20180131`
--  `--timerange=20180101-`
-- `--timerange=20180101-20181231`
 
 
 **Update testdata directory**

freqtrade/arguments.py

@@ -6,6 +6,7 @@ import argparse
 import logging
 import os
 import re
+import arrow
 from typing import List, Tuple, Optional
 
 from freqtrade import __version__
@@ -228,12 +229,16 @@ class Arguments(object):
                 stop = None
                 if stype[0]:
                     start = rvals[index]
-                    if stype[0] != 'date':
+                    if stype[0] == 'date':
+                        start = arrow.get(start, 'YYYYMMDD').timestamp
+                    else:
                         start = int(start)
                     index += 1
                 if stype[1]:
                     stop = rvals[index]
-                    if stype[1] != 'date':
+                    if stype[1] == 'date':
+                        stop = arrow.get(stop, 'YYYYMMDD').timestamp
+                    else:
                         stop = int(stop)
                 return stype, start, stop
         raise Exception('Incorrect syntax for timerange "%s"' % text)

freqtrade/exchange/__init__.py

@@ -271,7 +271,26 @@ def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict:
 @retrier
 def get_ticker_history(pair: str, tick_interval: str, since: Optional[int] = None) -> List[Dict]:
     try:
-        return _API.fetch_ohlcv(pair, timeframe=tick_interval, since=since)
+        # download data until it reaches today now time
+        #
+        # it looks as if some exchanges return cached data
+        # and update it with some delay so 10 mins interval is added
+        data = []
+        while not since or since < arrow.utcnow().shift(minutes=-10).timestamp * 1000:
+            data_part = _API.fetch_ohlcv(pair, timeframe=tick_interval, since=since)
+
+            if not data_part:
+                break
+
+            logger.info('Downloaded data for time range [%s, %s]',
+                        arrow.get(data_part[0][0] / 1000).format(),
+                        arrow.get(data_part[-1][0] / 1000).format())
+
+            data.extend(data_part)
+            since = data[-1][0] + 1
+
+        return data
+
     except ccxt.NetworkError as e:
         raise NetworkException(
             'Could not load ticker history due to networking error. Message: {}'.format(e)

freqtrade/optimize/__init__.py

@@ -4,10 +4,13 @@ import gzip
 import json
 import logging
 import os
+import arrow
 from typing import Optional, List, Dict, Tuple
 
 from freqtrade import misc
 from freqtrade.exchange import get_ticker_history
+from freqtrade.constants import Constants
+
 from user_data.hyperopt_conf import hyperopt_optimize_conf
 
 logger = logging.getLogger(__name__)
@@ -15,14 +18,30 @@ logger = logging.getLogger(__name__)
 
 def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]:
     stype, start, stop = timerange
-    if stype == (None, 'line'):
-        return tickerlist[stop:]
-    elif stype == ('line', None):
-        return tickerlist[0:start]
-    elif stype == ('index', 'index'):
-        return tickerlist[start:stop]
 
-    return tickerlist
+    start_index = 0
+    stop_index = len(tickerlist)
+
+    if stype[0] == 'line':
+        stop_index = start
+    if stype[0] == 'index':
+        start_index = start
+    elif stype[0] == 'date':
+        while tickerlist[start_index][0] < start * 1000:
+            start_index += 1
+
+    if stype[1] == 'line':
+        start_index = len(tickerlist) + stop
+    if stype[1] == 'index':
+        stop_index = stop
+    elif stype[1] == 'date':
+        while tickerlist[stop_index-1][0] > stop * 1000:
+            stop_index -= 1
+
+    if start_index > stop_index:
+        raise ValueError(f'The timerange [{start},{stop}] is incorrect')
+
+    return tickerlist[start_index:stop_index]
 
 
 def load_tickerdata_file(
@@ -75,7 +94,7 @@ def load_data(datadir: str,
     # If the user force the refresh of pairs
     if refresh_pairs:
         logger.info('Download data for all pairs and store them in %s', datadir)
-        download_pairs(datadir, _pairs, ticker_interval)
+        download_pairs(datadir, _pairs, ticker_interval, timerange=timerange)
 
     for pair in _pairs:
         pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
@@ -97,11 +116,13 @@ def make_testdata_path(datadir: str) -> str:
     )
 
 
-def download_pairs(datadir, pairs: List[str], ticker_interval: str) -> bool:
+def download_pairs(datadir, pairs: List[str],
+                   ticker_interval: str,
+                   timerange: Optional[Tuple[Tuple, int, int]] = None) -> bool:
     """For each pairs passed in parameters, download the ticker intervals"""
     for pair in pairs:
         try:
-            download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
+            download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval, timerange=timerange)
         except BaseException:
             logger.info(
                 'Failed to download the pair: "%s", Interval: %s',
@@ -112,12 +133,31 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: str) -> bool:
     return True
 
 
+def get_start_ts_from_timerange(timerange: Tuple[Tuple, int, int], interval: str) -> int:
+    if not timerange:
+        return None
+
+    if timerange[0][0] == 'date':
+        return timerange[1] * 1000
+    
+    if timerange[0][1] == 'line':
+        num_minutes = timerange[2] * Constants.TICKER_INTERVAL_MINUTES[interval]
+        return arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000
+
+    return None
+
+
 # FIX: 20180110, suggest rename interval to tick_interval
-def download_backtesting_testdata(datadir: str, pair: str, interval: str = '5m') -> bool:
+def download_backtesting_testdata(datadir: str,
+                                  pair: str,
+                                  interval: str = '5m',
+                                  timerange: Optional[Tuple[Tuple, int, int]] = None) -> bool:
     """
-    Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
+    Download the latest ticker intervals from the exchange for the pairs passed in parameters
     Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
     :param pairs: list of pairs to download
+    :param interval: ticker interval 
+    :param timerange: range of time to download
     :return: bool
     """
 
@@ -134,23 +174,33 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: str = '5m')
         interval=interval,
     ))
 
+    since = get_start_ts_from_timerange(timerange, interval)
+
     if os.path.isfile(filename):
         with open(filename, "rt") as file:
             data = json.load(file)
-        logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]))
+
-        logger.debug("Current End: %s", misc.format_ms_time(data[-1:][0][0]))
+        if since:
+            if since < data[0][0]:
+                # fully update the data
+                data = []
+            else:
+                # download unexist data only
+                since = max(since, data[-1][0] + 1)
+        else:
+            # download unexist data only
+            since = data[-1][0] + 1
     else:
         data = []
-        logger.debug("Current Start: None")
-        logger.debug("Current End: None")
 
-    new_data = get_ticker_history(pair=pair, tick_interval=interval)
+    logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
-    for row in new_data:
+    logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
-        if row not in data:
+
-            data.append(row)
+    new_data = get_ticker_history(pair=pair, tick_interval=interval, since=since)
+    data.extend(new_data)
+
     logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
-    logger.debug("New End: %s", misc.format_ms_time(data[-1:][0][0]))
+    logger.debug("New End: %s", misc.format_ms_time(data[-1][0]))
-    data = sorted(data, key=lambda data: data[0])
 
     misc.file_dump_json(filename, data)
 

freqtrade/tests/exchange/test_exchange.py

@@ -321,6 +321,15 @@ def test_get_ticker(default_conf, mocker):
         get_ticker(pair='ETH/BTC', refresh=True)
 
 
+def make_fetch_ohlcv_mock(data):
+    def fetch_ohlcv_mock(pair, timeframe, since):
+        if since:
+            assert since > data[-1][0]
+            return []
+        return data
+    return fetch_ohlcv_mock
+
+
 def test_get_ticker_history(default_conf, mocker):
     api_mock = MagicMock()
     tick = [
@@ -334,7 +343,7 @@ def test_get_ticker_history(default_conf, mocker):
         ]
     ]
     type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True})
-    api_mock.fetch_ohlcv = MagicMock(return_value=tick)
+    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick))
     mocker.patch('freqtrade.exchange._API', api_mock)
 
     # retrieve original ticker
@@ -357,7 +366,7 @@ def test_get_ticker_history(default_conf, mocker):
             10,  # volume (in quote currency)
         ]
     ]
-    api_mock.fetch_ohlcv = MagicMock(return_value=new_tick)
+    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick))
     mocker.patch('freqtrade.exchange._API', api_mock)
 
     ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval'])

freqtrade/tests/optimize/test_optimize.py

@@ -3,12 +3,14 @@
 import json
 import os
 import uuid
+import arrow
 from shutil import copyfile
 
 from freqtrade import optimize
 from freqtrade.misc import file_dump_json
 from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \
-    download_backtesting_testdata, load_tickerdata_file, trim_tickerlist
+    download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \
+    get_start_ts_from_timerange
 from freqtrade.tests.conftest import log_has
 
 # Change this if modifying UNITTEST/BTC testdatafile
@@ -145,6 +147,28 @@ def test_download_pairs(ticker_history, mocker) -> None:
     _clean_test_file(file2_5)
 
 
+def test_get_start_ts_from_timerange(mocker) -> None:
+    start = get_start_ts_from_timerange(None, '1m')
+    assert start is None
+
+    # check 'date'
+    start = get_start_ts_from_timerange((('date', 'date'), 1000, 2000), '1m')
+    assert start == 1000 * 1000
+
+    start = get_start_ts_from_timerange((('date', 'date'), 1000, 2000), '5m')
+    assert start == 1000 * 1000
+
+    # check 'line'
+    mock_now = arrow.get(1367900664)
+    mocker.patch('arrow.utcnow', return_value=mock_now)
+
+    start = get_start_ts_from_timerange(((None, 'line'), None, -200), '1m')
+    assert start == (1367900664 - 200 * 60) * 1000
+
+    start = get_start_ts_from_timerange(((None, 'line'), None, -200), '5m')
+    assert start == (1367900664 - 5 * 200 * 60) * 1000
+    
+
 def test_download_pairs_exception(ticker_history, mocker, caplog) -> None:
     mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history)
     mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata',
@@ -221,12 +245,12 @@ def test_trim_tickerlist() -> None:
     ticker_list_len = len(ticker_list)
 
     # Test the pattern ^(-\d+)$
-    # This pattern remove X element from the beginning
+    # This pattern uses the latest N elements
-    timerange = ((None, 'line'), None, 5)
+    timerange = ((None, 'line'), None, -5)
     ticker = trim_tickerlist(ticker_list, timerange)
     ticker_len = len(ticker)
 
-    assert ticker_list_len == ticker_len + 5
+    assert ticker_len == 5
     assert ticker_list[0] is not ticker[0]  # The first element should be different
     assert ticker_list[-1] is ticker[-1]  # The last element must be the same
 
@@ -251,6 +275,37 @@ def test_trim_tickerlist() -> None:
     assert ticker_list[5] is ticker[0]  # The list starts at the index 5
     assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
 
+    # Test the pattern ^(\d{8})-(\d{8})$
+    # This pattern extract a window between the dates
+    timerange = (('date', 'date'), ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is not ticker[0]  # The first element should be different
+    assert ticker_list[5] is ticker[0]  # The list starts at the index 5
+    assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
+
+    # Test the pattern ^-(\d{8})$
+    # This pattern extracts elements from the start to the date
+    timerange = ((None, 'date'), None, ticker_list[10][0] / 1000 - 1)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 10
+    assert ticker_list[0] is ticker[0]  # The start of the list is included
+    assert ticker_list[9] is ticker[-1]  # The element 10 is not included
+
+    # Test the pattern ^(\d{8})-$
+    # This pattern extracts elements from the date to now
+    timerange = (('date', None), ticker_list[10][0] / 1000 - 1, None)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == ticker_list_len - 10
+    assert ticker_list[10] is ticker[0]  # The first element is element #10
+    assert ticker_list[-1] is ticker[-1]  # The last element is the same
+
     # Test a wrong pattern
     # This pattern must return the list unchanged
     timerange = ((None, None), None, 5)

freqtrade/tests/test_arguments.py

@@ -89,6 +89,12 @@ def test_parse_args_dynamic_whitelist_invalid_values() -> None:
 def test_parse_timerange_incorrect() -> None:
     assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200')
     assert (('line', None), 200, None) == Arguments.parse_timerange('200-')
+    assert (('index', 'index'), 200, 500) == Arguments.parse_timerange('200-500')
+
+    assert (('date', None), 1274486400, None) == Arguments.parse_timerange('20100522-')
+    assert ((None, 'date'), None, 1274486400) == Arguments.parse_timerange('-20100522')
+    assert (('date', 'date'), 1274486400, 1438214400) == Arguments.parse_timerange('20100522-20150730')
+
     with pytest.raises(Exception, match=r'Incorrect syntax.*'):
         Arguments.parse_timerange('-')
 

scripts/download_backtest_data.py

@@ -4,8 +4,9 @@
 import json
 import sys
 import os
-import time
+import arrow
-import datetime
+
+from freqtrade import (exchange, arguments, misc)
 
 DEFAULT_DL_PATH = 'freqtrade/tests/testdata'
 
@@ -15,9 +16,6 @@ args = arguments.parse_args()
 
 TICKER_INTERVALS = ['1m', '5m']
 PAIRS = []
-MIN_SECCONDS = 60
-HOUR_SECCONDS = 60 * MIN_SECCONDS
-DAY_SECCONDS = 24 * HOUR_SECCONDS
 
 if args.pairs_file:
     with open(args.pairs_file) as file:
@@ -28,6 +26,11 @@ dl_path = DEFAULT_DL_PATH
 if args.export and os.path.exists(args.export):
     dl_path = args.export
 
+since_time = None
+if args.days:
+    since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000
+
+
 print(f'About to download pairs: {PAIRS} to {dl_path}')
 
 # Init exchange
@@ -35,29 +38,19 @@ exchange._API = exchange.init_ccxt({'key': '',
                                     'secret': '',
                                     'name': args.exchange})
 
+
 for pair in PAIRS:
     for tick_interval in TICKER_INTERVALS:
         print(f'downloading pair {pair}, interval {tick_interval}')
 
-        since_time = None
+        data = exchange.get_ticker_history(pair, tick_interval, since=since_time)
-        if args.days:
+        if not data:
-            since_time = int((time.time() - args.days * DAY_SECCONDS) * 1000)
+            print('\tNo data was downloaded')
+            break
             
-        # download data until it reaches today now time
+        print('\tData was downloaded for period %s - %s' % (
-        data = []
+            arrow.get(data[0][0] / 1000).format(),
-        while not since_time or since_time < (time.time() - 10 * MIN_SECCONDS) * 1000:
+            arrow.get(data[-1][0] / 1000).format()))
-            data_part = exchange.get_ticker_history(pair, tick_interval, since=since_time)
-
-            if not data_part:
-                print('\tNo data since %s' % datetime.datetime.utcfromtimestamp(since_time / 1000).strftime('%Y-%m-%dT%H:%M:%S'))
-                break
-            
-            print('\tData received for period %s - %s' % 
-                    (datetime.datetime.utcfromtimestamp(data_part[0][0] / 1000).strftime('%Y-%m-%dT%H:%M:%S'),
-                    datetime.datetime.utcfromtimestamp(data_part[-1][0] / 1000).strftime('%Y-%m-%dT%H:%M:%S')))
-
-            data.extend(data_part)
-            since_time = data[-1][0] + 1
 
         # save data
         pair_print = pair.replace('/', '_')