adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move Backtesting to a class and add unit tests

Browse Source
This commit is contained in:
Gerald Lonlas
2018-02-08 23:35:38 -08:00
parent db67b10605
commit 1d251d6151
9 changed files with 942 additions and 427 deletions
Download Patch File Download Diff File Expand all files Collapse all files

447
freqtrade/tests/optimize/test_backtesting.py
View File

@@ -1,14 +1,24 @@
# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103
# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
import logging
import json
import math
from typing import List
from copy import deepcopy
from unittest.mock import MagicMock
import pandas as pd
from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex
from freqtrade.optimize import preprocess
from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe
import freqtrade.optimize.backtesting as backtesting
from freqtrade import optimize
from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration
from freqtrade.arguments import Arguments
from freqtrade.analyze import Analyze
import freqtrade.tests.conftest as tt # test tools
# Avoid to reinit the same object again and again
_BACKTESTING = Backtesting(tt.default_conf())
def get_args(args) -> List[str]:
return Arguments(args, '').get_parsed_arg()
def trim_dictlist(dict_list, num):
@@ -18,60 +28,6 @@ def trim_dictlist(dict_list, num):
return new
def test_generate_text_table():
results = pd.DataFrame(
{
'currency': ['BTC_ETH', 'BTC_ETH'],
'profit_percent': [0.1, 0.2],
'profit_BTC': [0.2, 0.4],
'duration': [10, 30],
'profit': [2, 0],
'loss': [0, 0]
}
)
print(generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5))
assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == (
'pair buy count avg profit % total profit BTC avg duration profit loss\n' # noqa
'------- ----------- -------------- ------------------ -------------- -------- ------\n' # noqa
'BTC_ETH 2 15.00 0.60000000 100.0 2 0\n' # noqa
'TOTAL 2 15.00 0.60000000 100.0 2 0') # noqa
def test_get_timeframe(default_strategy):
data = preprocess(optimize.load_data(
None, ticker_interval=1, pairs=['BTC_UNITEST']))
min_date, max_date = get_timeframe(data)
assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
assert max_date.isoformat() == '2017-11-14T22:59:00+00:00'
def test_backtest(default_strategy, default_conf, mocker):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH'])
data = trim_dictlist(data, -200)
results = backtest({'stake_amount': default_conf['stake_amount'],
'processed': optimize.preprocess(data),
'max_open_trades': 10,
'realistic': True})
assert not results.empty
def test_backtest_1min_ticker_interval(default_strategy, default_conf, mocker):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
# Run a backtesting for an exiting 5min ticker_interval
data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'])
data = trim_dictlist(data, -200)
results = backtest({'stake_amount': default_conf['stake_amount'],
'processed': optimize.preprocess(data),
'max_open_trades': 1,
'realistic': True})
assert not results.empty
def load_data_test(what):
timerange = ((None, 'line'), None, -100)
data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'], timerange=timerange)
@@ -115,79 +71,324 @@ def load_data_test(what):
return data
def simple_backtest(config, contour, num_results):
def simple_backtest(config, contour, num_results) -> None:
backtesting = _BACKTESTING
data = load_data_test(contour)
processed = optimize.preprocess(data)
processed = backtesting.tickerdata_to_dataframe(data)
assert isinstance(processed, dict)
results = backtest({'stake_amount': config['stake_amount'],
'processed': processed,
'max_open_trades': 1,
'realistic': True})
results = backtesting.backtest(
{
'stake_amount': config['stake_amount'],
'processed': processed,
'max_open_trades': 1,
'realistic': True
}
)
# results :: <class 'pandas.core.frame.DataFrame'>
assert len(results) == num_results
# Test backtest on offline data
# loaded by freqdata/optimize/__init__.py::load_data()
def test_backtest2(default_conf, mocker, default_strategy):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH'])
data = trim_dictlist(data, -200)
results = backtest({'stake_amount': default_conf['stake_amount'],
'processed': optimize.preprocess(data),
'max_open_trades': 10,
'realistic': True})
assert not results.empty
def test_processed(default_conf, mocker, default_strategy):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
dict_of_tickerrows = load_data_test('raise')
dataframes = optimize.preprocess(dict_of_tickerrows)
dataframe = dataframes['BTC_UNITEST']
cols = dataframe.columns
# assert the dataframe got some of the indicator columns
for col in ['close', 'high', 'low', 'open', 'date',
'ema50', 'ao', 'macd', 'plus_dm']:
assert col in cols
def test_backtest_pricecontours(default_conf, mocker, default_strategy):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
tests = [['raise', 17], ['lower', 0], ['sine', 17]]
for [contour, numres] in tests:
simple_backtest(default_conf, contour, numres)
def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False, timerange=None):
tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1, timerange=timerange)
pairdata = {'BTC_UNITEST': tickerdata}
return pairdata
def test_backtest_start(default_conf, mocker, caplog):
caplog.set_level(logging.INFO)
default_conf['exchange']['pair_whitelist'] = ['BTC_UNITEST']
mocker.patch.dict('freqtrade.main._CONF', default_conf)
mocker.patch('freqtrade.misc.load_config', new=lambda s: default_conf)
mocker.patch.multiple('freqtrade.optimize',
load_data=mocked_load_data)
args = MagicMock()
args.ticker_interval = 1
args.level = 10
args.live = False
args.datadir = None
args.export = None
args.timerange = '-100' # needed due to MagicMock malleability
backtesting.start(args)
# Unit tests
def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None:
"""
Test setup_configuration() function
"""
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
read_data=json.dumps(default_conf)
))
args = [
'--config', 'config.json',
'--strategy', 'default_strategy',
'backtesting'
]
config = setup_configuration(get_args(args))
assert 'max_open_trades' in config
assert 'stake_currency' in config
assert 'stake_amount' in config
assert 'exchange' in config
assert 'pair_whitelist' in config['exchange']
assert 'datadir' in config
assert tt.log_has(
'Parameter --datadir detected: {} ...'.format(config['datadir']),
caplog.record_tuples
)
assert 'ticker_interval' in config
assert not tt.log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert 'live' not in config
assert not tt.log_has('Parameter -l/--live detected ...', caplog.record_tuples)
assert 'realistic_simulation' not in config
assert not tt.log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples)
assert 'refresh_pairs' not in config
assert not tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
assert 'timerange' not in config
assert 'export' not in config
def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None:
"""
Test setup_configuration() function
"""
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
read_data=json.dumps(default_conf)
))
args = [
'--config', 'config.json',
'--strategy', 'default_strategy',
'--datadir', '/foo/bar',
'backtesting',
'--ticker-interval', '1',
'--live',
'--realistic-simulation',
'--refresh-pairs-cached',
'--timerange', ':100',
'--export', '/bar/foo'
]
config = setup_configuration(get_args(args))
assert 'max_open_trades' in config
assert 'stake_currency' in config
assert 'stake_amount' in config
assert 'exchange' in config
assert 'pair_whitelist' in config['exchange']
assert 'datadir' in config
assert tt.log_has(
'Parameter --datadir detected: {} ...'.format(config['datadir']),
caplog.record_tuples
)
assert 'ticker_interval' in config
assert tt.log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert tt.log_has(
'Using ticker_interval: 1 ...',
caplog.record_tuples
)
assert 'live' in config
assert tt.log_has('Parameter -l/--live detected ...', caplog.record_tuples)
assert 'realistic_simulation'in config
assert tt.log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples)
assert tt.log_has('Using max_open_trades: 1 ...', caplog.record_tuples)
assert 'refresh_pairs'in config
assert tt.log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
import pprint
pprint.pprint(caplog.record_tuples)
pprint.pprint(config['timerange'])
assert 'timerange' in config
assert tt.log_has(
'Parameter --timerange detected: {} ...'.format(config['timerange']),
caplog.record_tuples
)
assert 'export' in config
assert tt.log_has(
'Parameter --export detected: {} ...'.format(config['export']),
caplog.record_tuples
)
def test_start(mocker, default_conf, caplog) -> None:
"""
Test start() function
"""
start_mock = MagicMock()
mocker.patch('freqtrade.optimize.backtesting.Backtesting.start', start_mock)
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
read_data=json.dumps(default_conf)
))
args = [
'--config', 'config.json',
'--strategy', 'default_strategy',
'backtesting'
]
args = get_args(args)
start(args)
assert tt.log_has(
'Starting freqtrade in Backtesting mode',
caplog.record_tuples
)
assert start_mock.call_count == 1
def test_backtesting__init__(mocker, default_conf) -> None:
"""
Test Backtesting.__init__() method
"""
init_mock = MagicMock()
mocker.patch('freqtrade.optimize.backtesting.Backtesting._init', init_mock)
backtesting = Backtesting(default_conf)
assert backtesting.config == default_conf
assert backtesting.analyze is None
assert backtesting.ticker_interval is None
assert backtesting.tickerdata_to_dataframe is None
assert backtesting.populate_buy_trend is None
assert backtesting.populate_sell_trend is None
assert init_mock.call_count == 1
def test_backtesting_init(default_conf) -> None:
"""
Test Backtesting._init() method
"""
backtesting = Backtesting(default_conf)
assert backtesting.config == default_conf
assert isinstance(backtesting.analyze, Analyze)
assert backtesting.ticker_interval == 5
assert callable(backtesting.tickerdata_to_dataframe)
assert callable(backtesting.populate_buy_trend)
assert callable(backtesting.populate_sell_trend)
def test_get_timeframe() -> None:
"""
Test Backtesting.get_timeframe() method
"""
backtesting = _BACKTESTING
data = backtesting.tickerdata_to_dataframe(
optimize.load_data(
None,
ticker_interval=1,
pairs=['BTC_UNITEST']
)
)
min_date, max_date = backtesting.get_timeframe(data)
assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
assert max_date.isoformat() == '2017-11-14T22:59:00+00:00'
def test_generate_text_table():
"""
Test Backtesting.generate_text_table() method
"""
backtesting = _BACKTESTING
results = pd.DataFrame(
{
'currency': ['BTC_ETH', 'BTC_ETH'],
'profit_percent': [0.1, 0.2],
'profit_BTC': [0.2, 0.4],
'duration': [10, 30],
'profit': [2, 0],
'loss': [0, 0]
}
)
result_str = (
'pair buy count avg profit % '
'total profit BTC avg duration profit loss\n'
'------- ----------- -------------- '
'------------------ -------------- -------- ------\n'
'BTC_ETH 2 15.00 '
'0.60000000 100.0 2 0\n'
'TOTAL 2 15.00 '
'0.60000000 100.0 2 0'
)
assert backtesting._generate_text_table(data={'BTC_ETH': {}}, results=results) == result_str
def test_backtesting_start(default_conf, mocker, caplog) -> None:
"""
Test Backtesting.start() method
"""
mocker.patch.multiple('freqtrade.optimize', load_data=mocked_load_data)
mocker.patch('freqtrade.exchange.get_ticker_history', MagicMock)
conf = deepcopy(default_conf)
conf['exchange']['pair_whitelist'] = ['BTC_UNITEST']
conf['ticker_interval'] = 1
conf['live'] = False
conf['datadir'] = None
conf['export'] = None
conf['timerange'] = '-100'
backtesting = Backtesting(conf)
backtesting.start()
# check the logs, that will contain the backtest result
exists = ['Using max_open_trades: 1 ...',
'Using stake_amount: 0.001 ...',
'Measuring data from 2017-11-14T21:17:00+00:00 '
'up to 2017-11-14T22:59:00+00:00 (0 days)..']
exists = [
'Using local backtesting data (using whitelist in given config) ...',
'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...',
'Measuring data from 2017-11-14T21:17:00+00:00 '
'up to 2017-11-14T22:59:00+00:00 (0 days)..'
]
for line in exists:
assert ('freqtrade.optimize.backtesting',
logging.INFO,
line) in caplog.record_tuples
assert tt.log_has(line, caplog.record_tuples)
def test_backtest(default_conf) -> None:
"""
Test Backtesting.backtest() method
"""
backtesting = _BACKTESTING
data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH'])
data = trim_dictlist(data, -200)
results = backtesting.backtest(
{
'stake_amount': default_conf['stake_amount'],
'processed': backtesting.tickerdata_to_dataframe(data),
'max_open_trades': 10,
'realistic': True
}
)
assert not results.empty
def test_backtest_1min_ticker_interval(default_conf) -> None:
"""
Test Backtesting.backtest() method with 1 min ticker
"""
backtesting = _BACKTESTING
# Run a backtesting for an exiting 5min ticker_interval
data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'])
data = trim_dictlist(data, -200)
results = backtesting.backtest(
{
'stake_amount': default_conf['stake_amount'],
'processed': backtesting.tickerdata_to_dataframe(data),
'max_open_trades': 1,
'realistic': True
}
)
assert not results.empty
def test_processed() -> None:
"""
Test Backtesting.backtest() method with offline data
"""
backtesting = _BACKTESTING
dict_of_tickerrows = load_data_test('raise')
dataframes = backtesting.tickerdata_to_dataframe(dict_of_tickerrows)
dataframe = dataframes['BTC_UNITEST']
cols = dataframe.columns
# assert the dataframe got some of the indicator columns
for col in ['close', 'high', 'low', 'open', 'date',
'ema50', 'ao', 'macd', 'plus_dm']:
assert col in cols
def test_backtest_pricecontours(default_conf) -> None:
tests = [['raise', 17], ['lower', 0], ['sine', 17]]
for [contour, numres] in tests:
simple_backtest(default_conf, contour, numres)

87
freqtrade/tests/optimize/test_optimize.py
View File

@@ -5,10 +5,11 @@ import json
import logging
import uuid
from shutil import copyfile
from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex
from freqtrade import optimize
from freqtrade.analyze import Analyze
from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\
download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, file_dump_json
download_backtesting_testdata, load_tickerdata_file, trim_tickerlist
from freqtrade.misc import file_dump_json
# Change this if modifying BTC_UNITEST testdatafile
_BTC_UNITTEST_LENGTH = 13681
@@ -45,12 +46,11 @@ def _clean_test_file(file: str) -> None:
os.rename(file_swp, file)
def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog):
caplog.set_level(logging.INFO)
def test_load_data_30min_ticker(ticker_history, mocker, caplog) -> None:
"""
Test load_data() with 30 min ticker
"""
mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history)
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
file = 'freqtrade/tests/testdata/BTC_UNITTEST-30.json'
_backup_file(file, copy_file=True)
@@ -62,12 +62,11 @@ def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog):
_clean_test_file(file)
def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog):
caplog.set_level(logging.INFO)
def test_load_data_5min_ticker(ticker_history, mocker, caplog) -> None:
"""
Test load_data() with 5 min ticker
"""
mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history)
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
file = 'freqtrade/tests/testdata/BTC_ETH-5.json'
_backup_file(file, copy_file=True)
@@ -79,12 +78,11 @@ def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog):
_clean_test_file(file)
def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog):
caplog.set_level(logging.INFO)
def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
"""
Test load_data() with 1 min ticker
"""
mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history)
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
file = 'freqtrade/tests/testdata/BTC_ETH-1.json'
_backup_file(file, copy_file=True)
@@ -96,12 +94,11 @@ def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog):
_clean_test_file(file)
def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, caplog):
caplog.set_level(logging.INFO)
def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None:
"""
Test load_data() with 1 min ticker
"""
mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history)
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
file = 'freqtrade/tests/testdata/BTC_MEME-1.json'
_backup_file(file)
@@ -113,14 +110,12 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl
_clean_test_file(file)
def test_testdata_path():
def test_testdata_path() -> None:
assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None)
def test_download_pairs(default_conf, ticker_history, mocker):
def test_download_pairs(ticker_history, mocker) -> None:
mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history)
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json'
file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json'
@@ -157,13 +152,10 @@ def test_download_pairs(default_conf, ticker_history, mocker):
_clean_test_file(file2_5)
def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog):
caplog.set_level(logging.INFO)
def test_download_pairs_exception(ticker_history, mocker, caplog) -> None:
mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history)
mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata',
side_effect=BaseException('File Error'))
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json'
file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json'
@@ -179,10 +171,8 @@ def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog):
'Failed to download the pair: "BTC-MEME", Interval: 1 min') in caplog.record_tuples
def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
def test_download_backtesting_testdata(ticker_history, mocker) -> None:
mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history)
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
# Download a 1 min ticker file
file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json'
@@ -200,7 +190,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
_clean_test_file(file2)
def test_download_backtesting_testdata2(mocker):
def test_download_backtesting_testdata2(mocker) -> None:
tick = [{'T': 'bar'}, {'T': 'foo'}]
mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick)
@@ -208,7 +198,7 @@ def test_download_backtesting_testdata2(mocker):
assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3)
def test_load_tickerdata_file():
def test_load_tickerdata_file() -> None:
# 7 does not exist in either format.
assert not load_tickerdata_file(None, 'BTC_UNITEST', 7)
# 1 exists only as a .json
@@ -219,22 +209,28 @@ def test_load_tickerdata_file():
assert _BTC_UNITTEST_LENGTH == len(tickerdata)
def test_init(default_conf, mocker):
def test_init(default_conf, mocker) -> None:
conf = {'exchange': {'pair_whitelist': []}}
mocker.patch('freqtrade.optimize.hyperopt_optimize_conf', return_value=conf)
assert {} == optimize.load_data('', pairs=[], refresh_pairs=True,
ticker_interval=int(default_conf['ticker_interval']))
assert {} == optimize.load_data(
'',
pairs=[],
refresh_pairs=True,
ticker_interval=int(default_conf['ticker_interval'])
)
def test_tickerdata_to_dataframe():
def test_tickerdata_to_dataframe(default_conf) -> None:
analyze = Analyze(default_conf)
timerange = ((None, 'line'), None, -100)
tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange)
tickerlist = {'BTC_UNITEST': tick}
data = optimize.tickerdata_to_dataframe(tickerlist)
data = analyze.tickerdata_to_dataframe(tickerlist)
assert len(data['BTC_UNITEST']) == 100
def test_trim_tickerlist():
def test_trim_tickerlist() -> None:
with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file:
ticker_list = json.load(data_file)
ticker_list_len = len(ticker_list)
@@ -279,8 +275,11 @@ def test_trim_tickerlist():
assert ticker_list_len == ticker_len
def test_file_dump_json():
def test_file_dump_json() -> None:
"""
Test file_dump_json()
:return: None
"""
file = 'freqtrade/tests/testdata/test_{id}.json'.format(id=str(uuid.uuid4()))
data = {'bar': 'foo'}