Merge pull request #107 from gcarq/feature/add-backtesting-subcommand

add backtesting subcommand and refresh test data

README.md

@@ -137,6 +137,43 @@ $ docker start freqtrade
 You do not need to rebuild the image for configuration
 changes, it will suffice to edit `config.json` and restart the container.
 
+### Usage
+```
+usage: freqtrade [-h] [-c PATH] [-v] [--version] [--dynamic-whitelist]
+                 {backtesting} ...
+
+Simple High Frequency Trading Bot for crypto currencies
+
+positional arguments:
+  {backtesting}
+    backtesting         backtesting module
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -c PATH, --config PATH
+                        specify configuration file (default: config.json)
+  -v, --verbose         be verbose
+  --version             show program's version number and exit
+  --dynamic-whitelist   dynamically generate and update whitelist based on 24h
+                        BaseVolume
+
+```
+
+### Backtesting
+
+Backtesting also uses the config specified via `-c/--config`.
+
+```
+usage: freqtrade backtesting [-h] [-l] [-i INT]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -l, --live            using live data
+  -i INT, --ticker-interval INT
+                        specify ticker interval in minutes (default: 5)
+```
+
+
 ### Execute tests
 
 ```

freqtrade/main.py

@@ -2,6 +2,7 @@
 import copy
 import json
 import logging
+import sys
 import time
 import traceback
 from datetime import datetime
@@ -10,13 +11,14 @@ from typing import Dict, Optional, List
 
 import requests
 from cachetools import cached, TTLCache
-from jsonschema import validate
 
 from freqtrade import __version__, exchange, persistence
 from freqtrade.analyze import get_signal, SignalType
 from freqtrade.misc import (
-    CONF_SCHEMA, State, get_state, update_state, build_arg_parser, throttle, FreqtradeException
+    FreqtradeException
 )
+from freqtrade.misc import State, get_state, update_state, parse_args, throttle, \
+    load_config
 from freqtrade.persistence import Trade
 from freqtrade.rpc import telegram
 
@@ -295,7 +297,9 @@ def main():
     :return: None
     """
     global _CONF
-    args = build_arg_parser().parse_args()
+    args = parse_args(sys.argv[1:])
+    if not args:
+        exit(0)
 
     # Initialize logger
     logging.basicConfig(
@@ -310,12 +314,7 @@ def main():
     )
 
     # Load and validate configuration
-    with open(args.config) as file:
-        _CONF = json.load(file)
-    if 'internals' not in _CONF:
-        _CONF['internals'] = {}
-    logger.info('Validating configuration ...')
-    validate(_CONF, CONF_SCHEMA)
+    _CONF = load_config(args.config)
 
     # Initialize all modules and start main loop
     if args.dynamic_whitelist:

freqtrade/misc.py

@@ -1,9 +1,12 @@
 import argparse
 import enum
+import json
 import logging
+import os
 import time
-from typing import Any, Callable
+from typing import Any, Callable, List, Dict
 
+from jsonschema import validate
 from wrapt import synchronized
 
 from freqtrade import __version__
@@ -44,6 +47,21 @@ def get_state() -> State:
     return _STATE
 
 
+def load_config(path: str) -> Dict:
+    """
+    Loads a config file from the given path
+    :param path: path as str
+    :return: configuration as dictionary
+    """
+    with open(path) as file:
+        conf = json.load(file)
+    if 'internals' not in conf:
+        conf['internals'] = {}
+    logger.info('Validating configuration ...')
+    validate(conf, CONF_SCHEMA)
+    return conf
+
+
 def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
     """
     Throttles the given callable that it
@@ -61,8 +79,11 @@ def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
     return result
 
 
-def build_arg_parser() -> argparse.ArgumentParser:
-    """ Builds and returns an ArgumentParser instance """
+def parse_args(args: List[str]):
+    """
+    Parses given arguments and returns an argparse Namespace instance.
+    Returns None if a sub command has been selected and executed.
+    """
     parser = argparse.ArgumentParser(
         description='Simple High Frequency Trading Bot for crypto currencies'
     )
@@ -92,7 +113,54 @@ def build_arg_parser() -> argparse.ArgumentParser:
         help='dynamically generate and update whitelist based on 24h BaseVolume',
         action='store_true',
     )
-    return parser
+    build_subcommands(parser)
+    parsed_args = parser.parse_args(args)
+
+    # No subcommand as been selected
+    if not hasattr(parsed_args, 'func'):
+        return parsed_args
+
+    parsed_args.func(parsed_args)
+    return None
+
+
+def build_subcommands(parser: argparse.ArgumentParser) -> None:
+    """ Builds and attaches all subcommands """
+    subparsers = parser.add_subparsers(dest='subparser')
+    backtest = subparsers.add_parser('backtesting', help='backtesting module')
+    backtest.set_defaults(func=start_backtesting)
+    backtest.add_argument(
+        '-l', '--live',
+        action='store_true',
+        dest='live',
+        help='using live data',
+    )
+    backtest.add_argument(
+        '-i', '--ticker-interval',
+        help='specify ticker interval in minutes (default: 5)',
+        dest='ticker_interval',
+        default=5,
+        type=int,
+        metavar='INT',
+    )
+
+
+def start_backtesting(args) -> None:
+    """
+    Exports all args as environment variables and starts backtesting via pytest.
+    :param args: arguments namespace
+    :return:
+    """
+    import pytest
+
+    os.environ.update({
+        'BACKTEST': 'true',
+        'BACKTEST_LIVE': 'true' if args.live else '',
+        'BACKTEST_CONFIG': args.config,
+        'BACKTEST_TICKER_INTERVAL': str(args.ticker_interval),
+    })
+    path = os.path.join(os.path.dirname(__file__), 'tests', 'test_backtesting.py')
+    pytest.main(['-s', path])
 
 
 # Required json-schema for user specified config

freqtrade/tests/__init__.py

@@ -0,0 +1,19 @@
+import json
+import os
+
+
+def load_backtesting_data(ticker_interval: int = 5):
+    path = os.path.abspath(os.path.dirname(__file__))
+    result = {}
+    pairs = [
+        'BTC_BCC', 'BTC_ETH', 'BTC_DASH', 'BTC_POWR', 'BTC_ETC',
+        'BTC_VTC', 'BTC_WAVES', 'BTC_LSK', 'BTC_XLM', 'BTC_OK',
+    ]
+    for pair in pairs:
+        with open('{abspath}/testdata/{pair}-{ticker_interval}.json'.format(
+                abspath=path,
+                pair=pair,
+                ticker_interval=ticker_interval,
+        )) as fp:
+            result[pair] = json.load(fp)
+    return result

freqtrade/tests/conftest.py

@@ -1,5 +1,4 @@
 # pragma pylint: disable=missing-docstring
-import json
 from datetime import datetime
 from unittest.mock import MagicMock
 
@@ -55,6 +54,8 @@ def default_conf():
 @pytest.fixture(scope="module")
 def backtest_conf():
     return {
+        "stake_currency": "BTC",
+        "stake_amount": 0.01,
         "minimal_roi": {
             "40":  0.0,
             "30":  0.01,
@@ -65,16 +66,6 @@ def backtest_conf():
     }
 
 
-@pytest.fixture(scope="module")
-def backdata():
-    result = {}
-    for pair in ['btc-neo', 'btc-eth', 'btc-omg', 'btc-edg', 'btc-pay',
-                 'btc-pivx', 'btc-qtum', 'btc-mtl', 'btc-etc', 'btc-ltc']:
-        with open('freqtrade/tests/testdata/' + pair + '.json') as data_file:
-            result[pair] = json.load(data_file)
-    return result
-
-
 @pytest.fixture
 def update():
     _update = Update(0)

freqtrade/tests/test_analyze.py

@@ -10,7 +10,7 @@ from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, popula
 
 @pytest.fixture
 def result():
-    with open('freqtrade/tests/testdata/btc-eth.json') as data_file:
+    with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file:
         return parse_ticker_dataframe(json.load(data_file))
 
 
@@ -20,7 +20,7 @@ def test_dataframe_correct_columns(result):
 
 
 def test_dataframe_correct_length(result):
-    assert len(result.index) == 5751
+    assert len(result.index) == 14382
 
 
 def test_populates_buy_trend(result):

freqtrade/tests/test_backtesting.py

@@ -1,30 +1,35 @@
 # pragma pylint: disable=missing-docstring
-from typing import Dict
+
+
 import logging
 import os
+from typing import Tuple, Dict
 
-import pytest
 import arrow
+import pytest
 from pandas import DataFrame
+from tabulate import tabulate
 
 from freqtrade import exchange
 from freqtrade.analyze import parse_ticker_dataframe, populate_indicators, \
     populate_buy_trend, populate_sell_trend
 from freqtrade.exchange import Bittrex
 from freqtrade.main import min_roi_reached
+from freqtrade.misc import load_config
 from freqtrade.persistence import Trade
+from freqtrade.tests import load_backtesting_data
 
-logging.disable(logging.DEBUG)  # disable debug logs that slow backtesting a lot
+logger = logging.getLogger(__name__)
 
 
-def format_results(results):
-    return 'Made {} buys. Average profit {:.2f}%. Total profit was {:.3f}. Average duration {:.1f} mins.'.format(
-        len(results.index), results.profit.mean() * 100.0, results.profit.sum(), results.duration.mean() * 5)
-
-
-def print_pair_results(pair, results):
-    print('For currency {}:'.format(pair))
-    print(format_results(results[results.currency == pair]))
+def format_results(results: DataFrame):
+    return 'Made {} buys. Average profit {:.2f}%. ' \
+           'Total profit was {:.3f}. Average duration {:.1f} mins.'.format(
+               len(results.index),
+               results.profit.mean() * 100.0,
+               results.profit.sum(),
+               results.duration.mean() * 5,
+           )
 
 
 def preprocess(backdata) -> Dict[str, DataFrame]:
@@ -34,11 +39,54 @@ def preprocess(backdata) -> Dict[str, DataFrame]:
     return processed
 
 
+def get_timeframe(data: Dict[str, Dict]) -> Tuple[arrow.Arrow, arrow.Arrow]:
+    """
+    Get the maximum timeframe for the given backtest data
+    :param data: dictionary with backtesting data
+    :return: tuple containing min_date, max_date
+    """
+    min_date, max_date = None, None
+    for values in data.values():
+        values = sorted(values, key=lambda d: arrow.get(d['T']))
+        if not min_date or values[0]['T'] < min_date:
+            min_date = values[0]['T']
+        if not max_date or values[-1]['T'] > max_date:
+            max_date = values[-1]['T']
+    return arrow.get(min_date), arrow.get(max_date)
+
+
+def generate_text_table(data: Dict[str, Dict], results: DataFrame, stake_currency) -> str:
+    """
+    Generates and returns a text table for the given backtest data and the results dataframe
+    :return: pretty printed table with tabulate as str
+    """
+    tabular_data = []
+    headers = ['pair', 'buy count', 'avg profit', 'total profit', 'avg duration']
+    for pair in data:
+        result = results[results.currency == pair]
+        tabular_data.append([
+            pair,
+            len(result.index),
+            '{:.2f}%'.format(result.profit.mean() * 100.0),
+            '{:.08f} {}'.format(result.profit.sum(), stake_currency),
+            '{:.2f}'.format(result.duration.mean() * 5),
+        ])
+
+    # Append Total
+    tabular_data.append([
+        'TOTAL',
+        len(results.index),
+        '{:.2f}%'.format(results.profit.mean() * 100.0),
+        '{:.08f} {}'.format(results.profit.sum(), stake_currency),
+        '{:.2f}'.format(results.duration.mean() * 5),
+    ])
+    return tabulate(tabular_data, headers=headers)
+
+
 def backtest(backtest_conf, processed, mocker):
     trades = []
     exchange._API = Bittrex({'key': '', 'secret': ''})
     mocker.patch.dict('freqtrade.main._CONF', backtest_conf)
-    mocker.patch('arrow.utcnow', return_value=arrow.get('2017-08-20T14:50:00'))
     for pair, pair_data in processed.items():
         pair_data['buy'] = 0
         pair_data['sell'] = 0
@@ -48,7 +96,7 @@ def backtest(backtest_conf, processed, mocker):
             trade = Trade(
                 open_rate=row.close,
                 open_date=row.date,
-                amount=1,
+                amount=backtest_conf['stake_amount'],
                 fee=exchange.get_fee() * 2
             )
             # calculate win/lose forwards from buy point
@@ -62,12 +110,45 @@ def backtest(backtest_conf, processed, mocker):
     return DataFrame.from_records(trades, columns=labels)
 
 
-@pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set")
-def test_backtest(backtest_conf, backdata, mocker, report=True):
-    results = backtest(backtest_conf, preprocess(backdata), mocker)
+@pytest.mark.skipif(not os.environ.get('BACKTEST'), reason="BACKTEST not set")
+def test_backtest(backtest_conf, mocker):
+    print('')
+    exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    print('====================== BACKTESTING REPORT ================================')
-    for pair in backdata:
-        print_pair_results(pair, results)
-    print('TOTAL OVER ALL TRADES:')
-    print(format_results(results))
+    # Load configuration file based on env variable
+    conf_path = os.environ.get('BACKTEST_CONFIG')
+    if conf_path:
+        print('Using config: {} ...'.format(conf_path))
+        config = load_config(conf_path)
+    else:
+        config = backtest_conf
+
+    # Parse ticker interval
+    ticker_interval = int(os.environ.get('BACKTEST_TICKER_INTERVAL') or 5)
+    print('Using ticker_interval: {} ...'.format(ticker_interval))
+
+    data = {}
+    if os.environ.get('BACKTEST_LIVE'):
+        print('Downloading data for all pairs in whitelist ...')
+        for pair in config['exchange']['pair_whitelist']:
+            data[pair] = exchange.get_ticker_history(pair, ticker_interval)
+    else:
+        print('Using local backtesting data (ignoring whitelist in given config)...')
+        data = load_backtesting_data(ticker_interval)
+
+    print('Using stake_currency: {} ...\nUsing stake_amount: {} ...'.format(
+        config['stake_currency'], config['stake_amount']
+    ))
+
+    # Print timeframe
+    min_date, max_date = get_timeframe(data)
+    print('Measuring data from {} up to {} ...'.format(
+        min_date.isoformat(), max_date.isoformat()
+    ))
+
+    # Execute backtest and print results
+    results = backtest(config, preprocess(data), mocker)
+    print('====================== BACKTESTING REPORT ======================================\n\n'
+          'NOTE: This Report doesn\'t respect the limits of max_open_trades, \n'
+          '      so the projected values should be taken with a grain of salt.\n')
+    print(generate_text_table(data, results, config['stake_currency']))

freqtrade/tests/test_hyperopt.py

@@ -9,7 +9,11 @@ import pytest
 from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
 from pandas import DataFrame
 
-from freqtrade.tests.test_backtesting import backtest, format_results, preprocess
+from freqtrade import exchange
+from freqtrade.exchange import Bittrex
+from freqtrade.tests import load_backtesting_data
+from freqtrade.tests.test_backtesting import backtest, format_results
+from freqtrade.tests.test_backtesting import preprocess
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
 logging.disable(logging.DEBUG)  # disable debug logs that slow backtesting a lot
@@ -19,6 +23,7 @@ TARGET_TRADES = 1300
 TOTAL_TRIES = 4
 current_tries = 0
 
+
 def buy_strategy_generator(params):
     def populate_buy_trend(dataframe: DataFrame) -> DataFrame:
         conditions = []
@@ -65,9 +70,12 @@ def buy_strategy_generator(params):
 
 
 @pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set")
-def test_hyperopt(backtest_conf, backdata, mocker):
+def test_hyperopt(backtest_conf, mocker):
     mocked_buy_trend = mocker.patch('freqtrade.tests.test_backtesting.populate_buy_trend')
+
+    backdata = load_backtesting_data()
     processed = preprocess(backdata)
+    exchange._API = Bittrex({'key': '', 'secret': ''})
 
     def optimizer(params):
         mocked_buy_trend.side_effect = buy_strategy_generator(params)

freqtrade/tests/test_misc.py

@@ -1,7 +1,12 @@
 # pragma pylint: disable=missing-docstring
 import time
+import os
+from argparse import Namespace
+from unittest.mock import MagicMock
 
-from freqtrade.misc import throttle
+import pytest
+
+from freqtrade.misc import throttle, parse_args, start_backtesting
 
 
 def test_throttle():
@@ -18,3 +23,99 @@ def test_throttle():
 
     result = throttle(func, -1)
     assert result == 42
+
+
+def test_parse_args_defaults():
+    args = parse_args([])
+    assert args is not None
+    assert args.config == 'config.json'
+    assert args.dynamic_whitelist is False
+    assert args.loglevel == 20
+
+
+def test_parse_args_invalid():
+    with pytest.raises(SystemExit, match=r'2'):
+        parse_args(['-c'])
+
+
+def test_parse_args_config():
+    args = parse_args(['-c', '/dev/null'])
+    assert args is not None
+    assert args.config == '/dev/null'
+
+    args = parse_args(['--config', '/dev/null'])
+    assert args is not None
+    assert args.config == '/dev/null'
+
+
+def test_parse_args_verbose():
+    args = parse_args(['-v'])
+    assert args is not None
+    assert args.loglevel == 10
+
+
+def test_parse_args_dynamic_whitelist():
+    args = parse_args(['--dynamic-whitelist'])
+    assert args is not None
+    assert args.dynamic_whitelist is True
+
+
+def test_parse_args_backtesting(mocker):
+    backtesting_mock = mocker.patch('freqtrade.misc.start_backtesting', MagicMock())
+    args = parse_args(['backtesting'])
+    assert args is None
+    assert backtesting_mock.call_count == 1
+
+    call_args = backtesting_mock.call_args[0][0]
+    assert call_args.config == 'config.json'
+    assert call_args.live is False
+    assert call_args.loglevel == 20
+    assert call_args.subparser == 'backtesting'
+    assert call_args.func is not None
+    assert call_args.ticker_interval == 5
+
+
+def test_parse_args_backtesting_invalid():
+    with pytest.raises(SystemExit, match=r'2'):
+        parse_args(['--ticker-interval'])
+
+    with pytest.raises(SystemExit, match=r'2'):
+        parse_args(['--ticker-interval', 'abc'])
+
+
+def test_parse_args_backtesting_custom(mocker):
+    backtesting_mock = mocker.patch('freqtrade.misc.start_backtesting', MagicMock())
+    args = parse_args(['-c', 'test_conf.json', 'backtesting', '--live', '--ticker-interval', '1'])
+    assert args is None
+    assert backtesting_mock.call_count == 1
+
+    call_args = backtesting_mock.call_args[0][0]
+    assert call_args.config == 'test_conf.json'
+    assert call_args.live is True
+    assert call_args.loglevel == 20
+    assert call_args.subparser == 'backtesting'
+    assert call_args.func is not None
+    assert call_args.ticker_interval == 1
+
+
+def test_start_backtesting(mocker):
+    pytest_mock = mocker.patch('pytest.main', MagicMock())
+    env_mock = mocker.patch('os.environ', {})
+    args = Namespace(
+        config='config.json',
+        live=True,
+        loglevel=20,
+        ticker_interval=1,
+    )
+    start_backtesting(args)
+    assert env_mock == {
+        'BACKTEST': 'true',
+        'BACKTEST_LIVE': 'true',
+        'BACKTEST_CONFIG': 'config.json',
+        'BACKTEST_TICKER_INTERVAL': '1',
+    }
+    assert pytest_mock.call_count == 1
+
+    main_call_args = pytest_mock.call_args[0][0]
+    assert main_call_args[0] == '-s'
+    assert main_call_args[1].endswith(os.path.join('freqtrade', 'tests', 'test_backtesting.py'))

freqtrade/tests/testdata/download_backtest_data.py

@@ -7,8 +7,13 @@ from os import path
 from freqtrade import exchange
 from freqtrade.exchange import Bittrex
 
-PAIRS = ['BTC-OK', 'BTC-NEO', 'BTC-DASH', 'BTC-ETC', 'BTC-ETH', 'BTC-SNT']
-TICKER_INTERVAL = 1  # ticker interval in minutes (currently implemented: 1 and 5)
+PAIRS = [
+    'BTC_BCC', 'BTC_ETH', 'BTC_MER', 'BTC_POWR', 'BTC_ETC',
+    'BTC_OK', 'BTC_NEO', 'BTC_EMC2', 'BTC_DASH', 'BTC_LSK',
+    'BTC_LTC', 'BTC_XZC', 'BTC_OMG', 'BTC_STRAT', 'BTC_XRP',
+    'BTC_QTUM', 'BTC_WAVES', 'BTC_VTC', 'BTC_XLM', 'BTC_MCO'
+]
+TICKER_INTERVAL = 5  # ticker interval in minutes (currently implemented: 1 and 5)
 OUTPUT_DIR = path.dirname(path.realpath(__file__))
 
 # Init Bittrex exchange
@@ -16,8 +21,8 @@ exchange._API = Bittrex({'key': '', 'secret': ''})
 
 for pair in PAIRS:
     data = exchange.get_ticker_history(pair, TICKER_INTERVAL)
-    filename = path.join(OUTPUT_DIR, '{}-{}m.json'.format(
-        pair.lower(),
+    filename = path.join(OUTPUT_DIR, '{}-{}.json'.format(
+        pair,
         TICKER_INTERVAL,
     ))
     with open(filename, 'w') as fp: