more advanced use of --timerange

docs/backtesting.md

@@ -57,9 +57,22 @@ you want to use. The last N ticks/timeframes will be used.
 Example:
 
 ```bash
-python3 ./freqtrade/main.py backtesting --timerange -200
+python3 ./freqtrade/main.py backtesting --timerange=-200
 ```
 
+***Advanced use of timerange***
+  Doing --timerange=-200 will get the last 200 timeframes
+  from your inputdata. You can also specify specific dates,
+  or a range span indexed by start and stop.
+  The full timerange specification:
+  Not implemented yet! --timerange=-20180131
+  Not implemented yet! --timerange=20180101-
+  Not implemented yet! --timerange=20180101-20181231
+  Last 123 tickframes of data:  --timerange=-123
+  First 123 tickframes of data: --timerange=123-
+  Tickframes from line 123 through 456: --timerange=123-456
+
+
 **Update testdata directory
 To update your testdata directory, or download into another testdata directory:
 ```bash

freqtrade/misc.py

@@ -4,6 +4,7 @@ import json
 import logging
 import time
 import os
+import re
 from typing import Any, Callable, Dict, List
 
 from jsonschema import Draft4Validator, validate
@@ -192,9 +193,9 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
     )
     backtesting_cmd.add_argument(
         '--timerange',
-        help='Use the last N ticks of data.',
+        help='Specify what timerange of data to use.',
         default=None,
-        type=int,
+        type=str,
         dest='timerange',
     )
 
@@ -224,14 +225,44 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
         metavar='INT',
     )
     hyperopt_cmd.add_argument(
-        '-tp', '--timerange',
+        '--timerange',
-        help='Use the last N ticks of data.',
+        help='Specify what timerange of data to use.',
         default=None,
-        type=int,
+        type=str,
         dest='timerange',
     )
 
 
+def parse_timerange(text):
+    if text is None:
+        return None
+    syntax = [('^-(\d{8})$',        (None,    'date')),
+              ('^(\d{8})-$',        ('date',  None)),
+              ('^(\d{8})-(\d{8})$', ('date',  'date')),
+              ('^(-\d+)$',          (None,    'line')),
+              ('^(\d+)-$',          ('line',  None)),
+              ('^(\d+)-(\d+)$',     ('index', 'index'))]
+    for rex, stype in syntax:
+        # Apply the regular expression to text
+        m = re.match(rex, text)
+        if m:  # Regex has matched
+            rvals = m.groups()
+            n = 0
+            start = None
+            stop = None
+            if stype[0]:
+                start = rvals[n]
+                if stype[0] != 'date':
+                    start = int(start)
+                n += 1
+            if stype[1]:
+                stop = rvals[n]
+                if stype[1] != 'date':
+                    stop = int(stop)
+            return (stype, start, stop)
+    raise Exception('Incorrect syntax for timerange "%s"' % text)
+
+
 # Required json-schema for user specified config
 CONF_SCHEMA = {
     'type': 'object',

freqtrade/optimize/__init__.py

@@ -12,14 +12,20 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
 logger = logging.getLogger(__name__)
 
 
-def trim_tickerlist(dl, num):
+def trim_tickerlist(tickerlist, timerange):
-    new = {}
+    (stype, start, stop) = timerange
-    for pair, pair_data in dl.items():
+    if stype == (None, 'line'):
-        new[pair] = pair_data[num:]
+        return tickerlist[stop:]
-    return new
+    elif stype == ('line', None):
+        return tickerlist[0:start]
+    elif stype == ('index', 'index'):
+        return tickerlist[start:stop]
+    else:
+        return tickerlist
 
 
-def load_tickerdata_file(datadir, pair, ticker_interval):
+def load_tickerdata_file(datadir, pair, ticker_interval,
+                         timerange=None):
     """
     Load a pair from file,
     :return dict OR empty if unsuccesful
@@ -37,11 +43,15 @@ def load_tickerdata_file(datadir, pair, ticker_interval):
     # Read the file, load the json
     with open(file) as tickerdata:
         pairdata = json.load(tickerdata)
+    if timerange:
+        pairdata = trim_tickerlist(pairdata, timerange)
     return pairdata
 
 
-def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]] = None,
+def load_data(datadir: str, ticker_interval: int = 5,
-              refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
+              pairs: Optional[List[str]] = None,
+              refresh_pairs: Optional[bool] = False,
+              timerange=None) -> Dict[str, List]:
     """
     Loads ticker history data for the given parameters
     :param ticker_interval: ticker interval in minutes
@@ -58,19 +68,17 @@ def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]]
         download_pairs(datadir, _pairs)
 
     for pair in _pairs:
-        pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
+        pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
         if not pairdata:
             # download the tickerdata from exchange
             download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
             # and retry reading the pair
-            pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
+            pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
         result[pair] = pairdata
     return result
 
 
-def tickerdata_to_dataframe(data, timerange=None):
+def tickerdata_to_dataframe(data):
-    if timerange:
-        data = trim_tickerlist(data, timerange)
     preprocessed = preprocess(data)
     return preprocessed
 

freqtrade/optimize/backtesting.py

@@ -163,9 +163,10 @@ def start(args):
         logger.info('Using stake_currency: %s ...', config['stake_currency'])
         logger.info('Using stake_amount: %s ...', config['stake_amount'])
 
+        timerange = misc.parse_timerange(args.timerange)
         data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval,
-                                  refresh_pairs=args.refresh_pairs)
+                                  refresh_pairs=args.refresh_pairs,
-
+                                  timerange=timerange)
     max_open_trades = 0
     if args.realistic_simulation:
         logger.info('Using max_open_trades: %s ...', config['max_open_trades'])
@@ -175,7 +176,7 @@ def start(args):
     from freqtrade import main
     main._CONF = config
 
-    preprocessed = optimize.tickerdata_to_dataframe(data, timerange=args.timerange)
+    preprocessed = optimize.tickerdata_to_dataframe(data)
     # Print timeframe
     min_date, max_date = get_timeframe(preprocessed)
     logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat())

freqtrade/optimize/hyperopt.py

@@ -15,7 +15,7 @@ from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe
 from hyperopt.mongoexp import MongoTrials
 from pandas import DataFrame
 
-from freqtrade import main  # noqa
+from freqtrade import main, misc  # noqa
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
 from freqtrade.misc import load_config
@@ -259,8 +259,11 @@ def start(args):
     logger.info('Using config: %s ...', args.config)
     config = load_config(args.config)
     pairs = config['exchange']['pair_whitelist']
-    data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval)
+    timerange = misc.parse_timerange(args.timerange)
-    PROCESSED = optimize.tickerdata_to_dataframe(data, timerange=args.timerange)
+    data = optimize.load_data(args.datadir, pairs=pairs,
+                              ticker_interval=args.ticker_interval,
+                              timerange=timerange)
+    PROCESSED = optimize.tickerdata_to_dataframe(data)
 
     if args.mongodb:
         logger.info('Using mongodb ...')

freqtrade/tests/optimize/test_backtesting.py

@@ -6,7 +6,7 @@ import pandas as pd
 from unittest.mock import MagicMock
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
-from freqtrade.optimize import preprocess, trim_tickerlist
+from freqtrade.optimize import preprocess
 from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe
 import freqtrade.optimize.backtesting as backtesting
 
@@ -60,8 +60,8 @@ def test_backtest_1min_ticker_interval(default_conf, mocker):
 
 
 def load_data_test(what):
-    data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'])
+    timerange = ((None, 'line'), None, -100)
-    data = trim_tickerlist(data, -100)
+    data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'], timerange=timerange)
     pair = data['BTC_UNITEST']
     datalen = len(pair)
     # Depending on the what parameter we now adjust the
@@ -142,10 +142,10 @@ def test_backtest_pricecontours(default_conf, mocker):
         simple_backtest(default_conf, contour, numres)
 
 
-def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False):
+def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False, timerange=None):
-    tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1)
+    tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1, timerange=timerange)
     pairdata = {'BTC_UNITEST': tickerdata}
-    return trim_tickerlist(pairdata, -100)
+    return pairdata
 
 
 def test_backtest_start(default_conf, mocker, caplog):
@@ -159,7 +159,7 @@ def test_backtest_start(default_conf, mocker, caplog):
     args.level = 10
     args.live = False
     args.datadir = None
-    args.timerange = None  # needed due to MagicMock malleability
+    args.timerange = '-100'  # needed due to MagicMock malleability
     backtesting.start(args)
     # check the logs, that will contain the backtest result
     exists = ['Using max_open_trades: 1 ...',

freqtrade/tests/optimize/test_hyperopt.py

@@ -62,7 +62,8 @@ def test_start_calls_fmin(mocker):
     mocker.patch('freqtrade.optimize.load_data')
     mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
 
-    args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False)
+    args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False,
+                       timerange=None)
     start(args)
 
     mock_fmin.assert_called_once()
@@ -75,7 +76,8 @@ def test_start_uses_mongotrials(mocker):
     mocker.patch('freqtrade.optimize.load_data')
     mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
 
-    args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True)
+    args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True,
+                       timerange=None)
     start(args)
 
     mock_mongotrials.assert_called_once()
@@ -129,7 +131,8 @@ def test_fmin_best_results(mocker, caplog):
     mocker.patch('freqtrade.optimize.load_data')
     mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result)
 
-    args = mocker.Mock(epochs=1, config='config.json.example')
+    args = mocker.Mock(epochs=1, config='config.json.example',
+                       timerange=None)
     start(args)
 
     exists = [
@@ -151,7 +154,8 @@ def test_fmin_throw_value_error(mocker, caplog):
     mocker.patch('freqtrade.optimize.load_data')
     mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError())
 
-    args = mocker.Mock(epochs=1, config='config.json.example')
+    args = mocker.Mock(epochs=1, config='config.json.example',
+                       timerange=None)
     start(args)
 
     exists = [
@@ -185,7 +189,8 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker):
                  return_value={})
     args = mocker.Mock(epochs=1,
                        config='config.json.example',
-                       mongodb=False)
+                       mongodb=False,
+                       timerange=None)
 
     start(args)
 

freqtrade/tests/optimize/test_optimize.py

@@ -177,7 +177,8 @@ def test_load_tickerdata_file():
 
 
 def test_tickerdata_to_dataframe():
-    tick = load_tickerdata_file(None, 'BTC_UNITEST', 1)
+    timerange = ((None, 'line'), None, -100)
+    tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange)
     tickerlist = {'BTC_UNITEST': tick}
-    data = optimize.tickerdata_to_dataframe(tickerlist, timerange=-100)
+    data = optimize.tickerdata_to_dataframe(tickerlist)
     assert 100 == len(data['BTC_UNITEST'])

freqtrade/tests/test_misc.py

@@ -8,7 +8,7 @@ import pytest
 from jsonschema import ValidationError
 
 from freqtrade.misc import (common_args_parser, load_config, parse_args,
-                            throttle)
+                            throttle, parse_timerange)
 
 
 def test_throttle():
@@ -133,6 +133,13 @@ def test_parse_args_hyperopt_custom(mocker):
     assert call_args.func is not None
 
 
+def test_parse_timerange_incorrect():
+    assert ((None, 'line'), None, -200) == parse_timerange('-200')
+    assert (('line', None), 200, None) == parse_timerange('200-')
+    with pytest.raises(Exception, match=r'Incorrect syntax.*'):
+        parse_timerange('-')
+
+
 def test_load_config(default_conf, mocker):
     file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open(
         read_data=json.dumps(default_conf)