From 512fcdbcb19f448dfeb46b7347a3053b46230f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rald=20LONLAS?= Date: Sat, 16 Dec 2017 06:42:28 -0800 Subject: [PATCH] Allow user to update testdata files with parameter --refresh-pairs-cached (#174) --- .gitignore | 3 + README.md | 19 +++- freqtrade/misc.py | 7 ++ freqtrade/optimize/__init__.py | 97 ++++++++++++++++++-- freqtrade/optimize/backtesting.py | 7 +- freqtrade/optimize/hyperopt.py | 10 +- freqtrade/tests/test_misc.py | 8 +- freqtrade/tests/test_optimize_backtesting.py | 76 +++++++++++++-- 8 files changed, 204 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 9b0606756..bcf06e9c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Freqtrade rules +freqtrade/tests/testdata/*.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index fa80c9e26..c7059732f 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ Backtesting also uses the config specified via `-c/--config`. ``` usage: freqtrade backtesting [-h] [-l] [-i INT] [--realistic-simulation] + [-r] optional arguments: -h, --help show this help message and exit @@ -201,9 +202,25 @@ optional arguments: --realistic-simulation uses max_open_trades from config to simulate real world limitations - + -r, --refresh-pairs-cached + refresh the pairs files in tests/testdata with + the latest data from Bittrex. Use it if you want + to run your backtesting with up-to-date data. ``` +#### How to use --refresh-pairs-cached parameter? +The first time your run Backtesting, it will take the pairs your have +set in your config file and download data from Bittrex. + +If for any reason you want to update your data set, you use +`--refresh-pairs-cached` to force Backtesting to update the data it has. +**Use it only if you want to update your data set. You will not be able +to come back to the previous version.** + +To test your strategy with latest data, we recommend to continue using +the parameter `-l` or `--live`. + + ### Hyperopt It is possible to use hyperopt for trading strategy optimization. diff --git a/freqtrade/misc.py b/freqtrade/misc.py index eff0c2228..33d87535c 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -166,6 +166,13 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None: action='store_true', dest='realistic_simulation', ) + backtesting_cmd.add_argument( + '-r', '--refresh-pairs-cached', + help='refresh the pairs files in tests/testdata with the latest data from Bittrex. Use it if you want to \ + run your backtesting with up-to-date data.', + action='store_true', + dest='refresh_pairs', + ) # Add hyperopt subcommand hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 12ec6330e..6880f709d 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,34 +1,46 @@ # pragma pylint: disable=missing-docstring - +import logging import json import os from typing import Optional, List, Dict +import time +from freqtrade.exchange import get_ticker_history from pandas import DataFrame from freqtrade.analyze import populate_indicators, parse_ticker_dataframe +logger = logging.getLogger(__name__) -def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None) -> Dict[str, List]: + +def load_data(pairs: List[str], ticker_interval: int = 5, refresh_pairs: Optional[bool] = False) -> Dict[str, List]: """ Loads ticker history data for the given parameters :param ticker_interval: ticker interval in minutes :param pairs: list of pairs :return: dict """ - path = os.path.abspath(os.path.dirname(__file__)) + path = testdata_path() result = {} - _pairs = pairs or [ - "BTC_ETH", "BTC_LTC", "BTC_ETC", "BTC_DASH", "BTC_ZEC", - "BTC_XLM", "BTC_NXT", "BTC_POWR", "BTC_ADA", "BTC_XMR", - ] - for pair in _pairs: - with open('{abspath}/../tests/testdata/{pair}-{ticker_interval}.json'.format( + + # If the user force the refresh of pairs + if refresh_pairs: + logger.info('Download data for all pairs and store them in freqtrade/tests/testsdata') + download_pairs(pairs) + + for pair in pairs: + file = '{abspath}/{pair}-{ticker_interval}.json'.format( abspath=path, pair=pair, ticker_interval=ticker_interval, - )) as tickerdata: + ) + # The file does not exist we download it + if not os.path.isfile(file): + download_backtesting_testdata(pair=pair, interval=ticker_interval) + + # Read the file, load the json + with open(file) as tickerdata: result[pair] = json.load(tickerdata) return result @@ -39,3 +51,68 @@ def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: for pair, pair_data in tickerdata.items(): processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data)) return processed + + +def testdata_path() -> str: + """Return the path where testdata files are stored""" + return os.path.abspath(os.path.dirname(__file__)) + '/../tests/testdata' + + +def download_pairs(pairs: List[str]) -> bool: + """For each pairs passed in parameters, download 1 and 5 ticker intervals""" + for pair in pairs: + try: + for interval in [1,5]: + download_backtesting_testdata(pair=pair, interval=interval) + except BaseException: + logger.info('Impossible to download the pair: "{pair}", Interval: {interval} min'.format( + pair=pair, + interval=interval, + )) + return False + return True + + +def download_backtesting_testdata(pair: str, interval: int = 5) -> bool: + """ + Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters + Based on @Rybolov work: https://github.com/rybolov/freqtrade-data + :param pairs: list of pairs to download + :return: bool + """ + + path = testdata_path() + logger.info('Download the pair: "{pair}", Interval: {interval} min'.format( + pair=pair, + interval=interval, + )) + + filepair = pair.replace("-", "_") + filename = os.path.join(path, '{}-{}.json'.format( + filepair, + interval, + )) + filename = filename.replace('USDT_BTC', 'BTC_FAKEBULL') + + if os.path.isfile(filename): + with open(filename, "rt") as fp: + data = json.load(fp) + logger.debug("Current Start:", data[1]['T']) + logger.debug("Current End: ", data[-1:][0]['T']) + else: + data = [] + logger.debug("Current Start: None") + logger.debug("Current End: None") + + new_data = get_ticker_history(pair = pair, tick_interval = int(interval)) + for row in new_data: + if row not in data: + data.append(row) + logger.debug("New Start:", data[1]['T']) + logger.debug("New End: ", data[-1:][0]['T']) + data = sorted(data, key=lambda data: data['T']) + + with open(filename, "wt") as fp: + json.dump(data, fp) + + return True diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5b34afd25..8b4d9eaef 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -132,13 +132,14 @@ def start(args): logger.info('Using ticker_interval: %s ...', args.ticker_interval) data = {} + pairs = config['exchange']['pair_whitelist'] if args.live: logger.info('Downloading data for all pairs in whitelist ...') - for pair in config['exchange']['pair_whitelist']: + for pair in pairs: data[pair] = exchange.get_ticker_history(pair, args.ticker_interval) else: - logger.info('Using local backtesting data (ignoring whitelist in given config) ...') - data = load_data(args.ticker_interval) + logger.info('Using local backtesting data (using whitelist in given config) ...') + data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs) logger.info('Using stake_currency: %s ...', config['stake_currency']) logger.info('Using stake_amount: %s ...', config['stake_amount']) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 78e5a3fb1..4cdfcd5c0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,6 +14,7 @@ from pandas import DataFrame from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex +from freqtrade.misc import load_config from freqtrade.optimize.backtesting import backtest from freqtrade.vendor.qtpylib.indicators import crossed_above @@ -34,7 +35,7 @@ AVG_PROFIT_TO_BEAT = 0.2 AVG_DURATION_TO_BEAT = 50 # Configuration and data used by hyperopt -PROCESSED = optimize.preprocess(optimize.load_data()) +PROCESSED = [] OPTIMIZE_CONFIG = { 'max_open_trades': 3, 'stake_currency': 'BTC', @@ -215,7 +216,7 @@ def buy_strategy_generator(params): def start(args): - global TOTAL_TRIES + global TOTAL_TRIES, PROCESSED TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -226,6 +227,11 @@ def start(args): format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', ) + logger.info('Using config: %s ...', args.config) + config = load_config(args.config) + pairs = config['exchange']['pair_whitelist'] + PROCESSED = optimize.preprocess(optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval)) + if args.mongodb: logger.info('Using mongodb ...') logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index ca3e69264..045d4f05e 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -95,7 +95,12 @@ def test_parse_args_backtesting_invalid(): def test_parse_args_backtesting_custom(mocker): backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock()) - args = parse_args(['-c', 'test_conf.json', 'backtesting', '--live', '--ticker-interval', '1']) + args = parse_args([ + '-c', 'test_conf.json', + 'backtesting', + '--live', + '--ticker-interval', '1', + '--refresh-pairs-cached']) assert args is None assert backtesting_mock.call_count == 1 @@ -106,6 +111,7 @@ def test_parse_args_backtesting_custom(mocker): assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval == 1 + assert call_args.refresh_pairs is True def test_parse_args_hyperopt(mocker): diff --git a/freqtrade/tests/test_optimize_backtesting.py b/freqtrade/tests/test_optimize_backtesting.py index d314214c3..fd4630fbb 100644 --- a/freqtrade/tests/test_optimize_backtesting.py +++ b/freqtrade/tests/test_optimize_backtesting.py @@ -4,6 +4,8 @@ from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.backtesting import backtest +from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata +import os import pytest @@ -13,8 +15,8 @@ def test_backtest(default_conf, mocker): data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) results = backtest(default_conf, optimize.preprocess(data), 10, True) - num_resutls = len(results) - assert num_resutls > 0 + num_results = len(results) + assert num_results > 0 def test_1min_ticker_interval(default_conf, mocker): @@ -26,7 +28,69 @@ def test_1min_ticker_interval(default_conf, mocker): results = backtest(default_conf, optimize.preprocess(data), 1, True) assert len(results) > 0 - # Run a backtesting for 5min ticker_interval - with pytest.raises(FileNotFoundError): - data = optimize.load_data(ticker_interval=5, pairs=['BTC_UNITEST']) - results = backtest(default_conf, optimize.preprocess(data), 1, True) +def test_backtest_with_new_pair(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + exchange._API = Bittrex({'key': '', 'secret': ''}) + + optimize.load_data(ticker_interval=1, pairs=['BTC_MEME']) + file = 'freqtrade/tests/testdata/BTC_MEME-1.json' + assert os.path.isfile(file) is True + + # delete file freshly downloaded + if os.path.isfile(file): + os.remove(file) + + +def test_testdata_path(): + assert str('freqtrade/optimize/../tests/testdata') in testdata_path() + + +def test_download_pairs(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + exchange._API = Bittrex({'key': '', 'secret': ''}) + + file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' + file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' + file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json' + file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json' + + assert download_pairs(pairs = ['BTC-MEME', 'BTC-CFI']) is True + + assert os.path.isfile(file1_1) is True + assert os.path.isfile(file1_5) is True + assert os.path.isfile(file2_1) is True + assert os.path.isfile(file2_5) is True + + # delete files freshly downloaded + if os.path.isfile(file1_1): + os.remove(file1_1) + + if os.path.isfile(file1_5): + os.remove(file1_5) + + if os.path.isfile(file2_1): + os.remove(file2_1) + + if os.path.isfile(file2_5): + os.remove(file2_5) + + +def test_download_backtesting_testdata(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + exchange._API = Bittrex({'key': '', 'secret': ''}) + + # Download a 1 min ticker file + file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' + download_backtesting_testdata(pair = "BTC-XEL", interval = 1) + assert os.path.isfile(file1) is True + + if os.path.isfile(file1): + os.remove(file1) + + # Download a 5 min ticker file + file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' + download_backtesting_testdata(pair = "BTC-STORJ", interval = 5) + assert os.path.isfile(file2) is True + + if os.path.isfile(file2): + os.remove(file2)