adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more advanced use of --timerange

Browse Source
This commit is contained in:
kryofly 2018-01-15 22:25:02 +01:00
parent 71bb348698
commit 0e58ab7e01
9 changed files with 109 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/backtesting.md
View File

@ -57,9 +57,22 @@ you want to use. The last N ticks/timeframes will be used.
Example:
```bash
python3 ./freqtrade/main.py backtesting --timerange -200
python3 ./freqtrade/main.py backtesting --timerange=-200
```
***Advanced use of timerange***
Doing --timerange=-200 will get the last 200 timeframes
from your inputdata. You can also specify specific dates,
or a range span indexed by start and stop.
The full timerange specification:
Not implemented yet! --timerange=-20180131
Not implemented yet! --timerange=20180101-
Not implemented yet! --timerange=20180101-20181231
Last 123 tickframes of data: --timerange=-123
First 123 tickframes of data: --timerange=123-
Tickframes from line 123 through 456: --timerange=123-456
**Update testdata directory
To update your testdata directory, or download into another testdata directory:
```bash

41
freqtrade/misc.py
View File

@ -4,6 +4,7 @@ import json
import logging
import time
import os
import re
from typing import Any, Callable, Dict, List
from jsonschema import Draft4Validator, validate
@ -192,9 +193,9 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
)
backtesting_cmd.add_argument(
'--timerange',
help='Use the last N ticks of data.',
help='Specify what timerange of data to use.',
default=None,
type=int,
type=str,
dest='timerange',
)
@ -224,14 +225,44 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
metavar='INT',
)
hyperopt_cmd.add_argument(
'-tp', '--timerange',
help='Use the last N ticks of data.',
'--timerange',
help='Specify what timerange of data to use.',
default=None,
type=int,
type=str,
dest='timerange',
)
def parse_timerange(text):
if text is None:
return None
syntax = [('^-(\d{8})$', (None, 'date')),
('^(\d{8})-$', ('date', None)),
('^(\d{8})-(\d{8})$', ('date', 'date')),
('^(-\d+)$', (None, 'line')),
('^(\d+)-$', ('line', None)),
('^(\d+)-(\d+)$', ('index', 'index'))]
for rex, stype in syntax:
# Apply the regular expression to text
m = re.match(rex, text)
if m: # Regex has matched
rvals = m.groups()
n = 0
start = None
stop = None
if stype[0]:
start = rvals[n]
if stype[0] != 'date':
start = int(start)
n += 1
if stype[1]:
stop = rvals[n]
if stype[1] != 'date':
stop = int(stop)
return (stype, start, stop)
raise Exception('Incorrect syntax for timerange "%s"' % text)
# Required json-schema for user specified config
CONF_SCHEMA = {
'type': 'object',

34
freqtrade/optimize/__init__.py
View File

@ -12,14 +12,20 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
logger = logging.getLogger(__name__)
def trim_tickerlist(dl, num):
new = {}
for pair, pair_data in dl.items():
new[pair] = pair_data[num:]
return new
def trim_tickerlist(tickerlist, timerange):
(stype, start, stop) = timerange
if stype == (None, 'line'):
return tickerlist[stop:]
elif stype == ('line', None):
return tickerlist[0:start]
elif stype == ('index', 'index'):
return tickerlist[start:stop]
else:
return tickerlist
def load_tickerdata_file(datadir, pair, ticker_interval):
def load_tickerdata_file(datadir, pair, ticker_interval,
timerange=None):
"""
Load a pair from file,
:return dict OR empty if unsuccesful
@ -37,11 +43,15 @@ def load_tickerdata_file(datadir, pair, ticker_interval):
# Read the file, load the json
with open(file) as tickerdata:
pairdata = json.load(tickerdata)
if timerange:
pairdata = trim_tickerlist(pairdata, timerange)
return pairdata
def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]] = None,
refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
def load_data(datadir: str, ticker_interval: int = 5,
pairs: Optional[List[str]] = None,
refresh_pairs: Optional[bool] = False,
timerange=None) -> Dict[str, List]:
"""
Loads ticker history data for the given parameters
:param ticker_interval: ticker interval in minutes
@ -58,19 +68,17 @@ def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]]
download_pairs(datadir, _pairs)
for pair in _pairs:
pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
if not pairdata:
# download the tickerdata from exchange
download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
# and retry reading the pair
pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
result[pair] = pairdata
return result
def tickerdata_to_dataframe(data, timerange=None):
if timerange:
data = trim_tickerlist(data, timerange)
def tickerdata_to_dataframe(data):
preprocessed = preprocess(data)
return preprocessed

7
freqtrade/optimize/backtesting.py
View File

@ -163,9 +163,10 @@ def start(args):
logger.info('Using stake_currency: %s ...', config['stake_currency'])
logger.info('Using stake_amount: %s ...', config['stake_amount'])
timerange = misc.parse_timerange(args.timerange)
data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval,
refresh_pairs=args.refresh_pairs)
refresh_pairs=args.refresh_pairs,
timerange=timerange)
max_open_trades = 0
if args.realistic_simulation:
logger.info('Using max_open_trades: %s ...', config['max_open_trades'])
@ -175,7 +176,7 @@ def start(args):
from freqtrade import main
main._CONF = config
preprocessed = optimize.tickerdata_to_dataframe(data, timerange=args.timerange)
preprocessed = optimize.tickerdata_to_dataframe(data)
# Print timeframe
min_date, max_date = get_timeframe(preprocessed)
logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat())

9
freqtrade/optimize/hyperopt.py
View File

@ -15,7 +15,7 @@ from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe
from hyperopt.mongoexp import MongoTrials
from pandas import DataFrame
from freqtrade import main # noqa
from freqtrade import main, misc # noqa
from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex
from freqtrade.misc import load_config
@ -259,8 +259,11 @@ def start(args):
logger.info('Using config: %s ...', args.config)
config = load_config(args.config)
pairs = config['exchange']['pair_whitelist']
data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval)
PROCESSED = optimize.tickerdata_to_dataframe(data, timerange=args.timerange)
timerange = misc.parse_timerange(args.timerange)
data = optimize.load_data(args.datadir, pairs=pairs,
ticker_interval=args.ticker_interval,
timerange=timerange)
PROCESSED = optimize.tickerdata_to_dataframe(data)
if args.mongodb:
logger.info('Using mongodb ...')

14
freqtrade/tests/optimize/test_backtesting.py
View File

@ -6,7 +6,7 @@ import pandas as pd
from unittest.mock import MagicMock
from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex
from freqtrade.optimize import preprocess, trim_tickerlist
from freqtrade.optimize import preprocess
from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe
import freqtrade.optimize.backtesting as backtesting
@ -60,8 +60,8 @@ def test_backtest_1min_ticker_interval(default_conf, mocker):
def load_data_test(what):
data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'])
data = trim_tickerlist(data, -100)
timerange = ((None, 'line'), None, -100)
data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'], timerange=timerange)
pair = data['BTC_UNITEST']
datalen = len(pair)
# Depending on the what parameter we now adjust the
@ -142,10 +142,10 @@ def test_backtest_pricecontours(default_conf, mocker):
simple_backtest(default_conf, contour, numres)
def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False):
tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1)
def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False, timerange=None):
tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1, timerange=timerange)
pairdata = {'BTC_UNITEST': tickerdata}
return trim_tickerlist(pairdata, -100)
return pairdata
def test_backtest_start(default_conf, mocker, caplog):
@ -159,7 +159,7 @@ def test_backtest_start(default_conf, mocker, caplog):
args.level = 10
args.live = False
args.datadir = None
args.timerange = None # needed due to MagicMock malleability
args.timerange = '-100' # needed due to MagicMock malleability
backtesting.start(args)
# check the logs, that will contain the backtest result
exists = ['Using max_open_trades: 1 ...',

15
freqtrade/tests/optimize/test_hyperopt.py
View File

@ -62,7 +62,8 @@ def test_start_calls_fmin(mocker):
mocker.patch('freqtrade.optimize.load_data')
mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False)
args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False,
timerange=None)
start(args)
mock_fmin.assert_called_once()
@ -75,7 +76,8 @@ def test_start_uses_mongotrials(mocker):
mocker.patch('freqtrade.optimize.load_data')
mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True)
args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True,
timerange=None)
start(args)
mock_mongotrials.assert_called_once()
@ -129,7 +131,8 @@ def test_fmin_best_results(mocker, caplog):
mocker.patch('freqtrade.optimize.load_data')
mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result)
args = mocker.Mock(epochs=1, config='config.json.example')
args = mocker.Mock(epochs=1, config='config.json.example',
timerange=None)
start(args)
exists = [
@ -151,7 +154,8 @@ def test_fmin_throw_value_error(mocker, caplog):
mocker.patch('freqtrade.optimize.load_data')
mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError())
args = mocker.Mock(epochs=1, config='config.json.example')
args = mocker.Mock(epochs=1, config='config.json.example',
timerange=None)
start(args)
exists = [
@ -185,7 +189,8 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker):
return_value={})
args = mocker.Mock(epochs=1,
config='config.json.example',
mongodb=False)
mongodb=False,
timerange=None)
start(args)

5
freqtrade/tests/optimize/test_optimize.py
View File

@ -177,7 +177,8 @@ def test_load_tickerdata_file():
def test_tickerdata_to_dataframe():
tick = load_tickerdata_file(None, 'BTC_UNITEST', 1)
timerange = ((None, 'line'), None, -100)
tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange)
tickerlist = {'BTC_UNITEST': tick}
data = optimize.tickerdata_to_dataframe(tickerlist, timerange=-100)
data = optimize.tickerdata_to_dataframe(tickerlist)
assert 100 == len(data['BTC_UNITEST'])

9
freqtrade/tests/test_misc.py
View File

@ -8,7 +8,7 @@ import pytest
from jsonschema import ValidationError
from freqtrade.misc import (common_args_parser, load_config, parse_args,
throttle)
throttle, parse_timerange)
def test_throttle():
@ -133,6 +133,13 @@ def test_parse_args_hyperopt_custom(mocker):
assert call_args.func is not None
def test_parse_timerange_incorrect():
assert ((None, 'line'), None, -200) == parse_timerange('-200')
assert (('line', None), 200, None) == parse_timerange('200-')
with pytest.raises(Exception, match=r'Incorrect syntax.*'):
parse_timerange('-')
def test_load_config(default_conf, mocker):
file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open(
read_data=json.dumps(default_conf)