Allow user to update testdata files with parameter --refresh-pairs-cached (#174)

This commit is contained in:
Gérald LONLAS 2017-12-16 06:42:28 -08:00 committed by Michael Egger
parent e00f02b603
commit 512fcdbcb1
8 changed files with 204 additions and 23 deletions

3
.gitignore vendored
View File

@ -1,3 +1,6 @@
# Freqtrade rules
freqtrade/tests/testdata/*.json
# Byte-compiled / optimized / DLL files # Byte-compiled / optimized / DLL files
__pycache__/ __pycache__/
*.py[cod] *.py[cod]

View File

@ -192,6 +192,7 @@ Backtesting also uses the config specified via `-c/--config`.
``` ```
usage: freqtrade backtesting [-h] [-l] [-i INT] [--realistic-simulation] usage: freqtrade backtesting [-h] [-l] [-i INT] [--realistic-simulation]
[-r]
optional arguments: optional arguments:
-h, --help show this help message and exit -h, --help show this help message and exit
@ -201,9 +202,25 @@ optional arguments:
--realistic-simulation --realistic-simulation
uses max_open_trades from config to simulate real uses max_open_trades from config to simulate real
world limitations world limitations
-r, --refresh-pairs-cached
refresh the pairs files in tests/testdata with
the latest data from Bittrex. Use it if you want
to run your backtesting with up-to-date data.
``` ```
#### How to use --refresh-pairs-cached parameter?
The first time your run Backtesting, it will take the pairs your have
set in your config file and download data from Bittrex.
If for any reason you want to update your data set, you use
`--refresh-pairs-cached` to force Backtesting to update the data it has.
**Use it only if you want to update your data set. You will not be able
to come back to the previous version.**
To test your strategy with latest data, we recommend to continue using
the parameter `-l` or `--live`.
### Hyperopt ### Hyperopt
It is possible to use hyperopt for trading strategy optimization. It is possible to use hyperopt for trading strategy optimization.

View File

@ -166,6 +166,13 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
action='store_true', action='store_true',
dest='realistic_simulation', dest='realistic_simulation',
) )
backtesting_cmd.add_argument(
'-r', '--refresh-pairs-cached',
help='refresh the pairs files in tests/testdata with the latest data from Bittrex. Use it if you want to \
run your backtesting with up-to-date data.',
action='store_true',
dest='refresh_pairs',
)
# Add hyperopt subcommand # Add hyperopt subcommand
hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')

View File

@ -1,34 +1,46 @@
# pragma pylint: disable=missing-docstring # pragma pylint: disable=missing-docstring
import logging
import json import json
import os import os
from typing import Optional, List, Dict from typing import Optional, List, Dict
import time
from freqtrade.exchange import get_ticker_history
from pandas import DataFrame from pandas import DataFrame
from freqtrade.analyze import populate_indicators, parse_ticker_dataframe from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
logger = logging.getLogger(__name__)
def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None) -> Dict[str, List]:
def load_data(pairs: List[str], ticker_interval: int = 5, refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
""" """
Loads ticker history data for the given parameters Loads ticker history data for the given parameters
:param ticker_interval: ticker interval in minutes :param ticker_interval: ticker interval in minutes
:param pairs: list of pairs :param pairs: list of pairs
:return: dict :return: dict
""" """
path = os.path.abspath(os.path.dirname(__file__)) path = testdata_path()
result = {} result = {}
_pairs = pairs or [
"BTC_ETH", "BTC_LTC", "BTC_ETC", "BTC_DASH", "BTC_ZEC", # If the user force the refresh of pairs
"BTC_XLM", "BTC_NXT", "BTC_POWR", "BTC_ADA", "BTC_XMR", if refresh_pairs:
] logger.info('Download data for all pairs and store them in freqtrade/tests/testsdata')
for pair in _pairs: download_pairs(pairs)
with open('{abspath}/../tests/testdata/{pair}-{ticker_interval}.json'.format(
for pair in pairs:
file = '{abspath}/{pair}-{ticker_interval}.json'.format(
abspath=path, abspath=path,
pair=pair, pair=pair,
ticker_interval=ticker_interval, ticker_interval=ticker_interval,
)) as tickerdata: )
# The file does not exist we download it
if not os.path.isfile(file):
download_backtesting_testdata(pair=pair, interval=ticker_interval)
# Read the file, load the json
with open(file) as tickerdata:
result[pair] = json.load(tickerdata) result[pair] = json.load(tickerdata)
return result return result
@ -39,3 +51,68 @@ def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
for pair, pair_data in tickerdata.items(): for pair, pair_data in tickerdata.items():
processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data)) processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data))
return processed return processed
def testdata_path() -> str:
"""Return the path where testdata files are stored"""
return os.path.abspath(os.path.dirname(__file__)) + '/../tests/testdata'
def download_pairs(pairs: List[str]) -> bool:
"""For each pairs passed in parameters, download 1 and 5 ticker intervals"""
for pair in pairs:
try:
for interval in [1,5]:
download_backtesting_testdata(pair=pair, interval=interval)
except BaseException:
logger.info('Impossible to download the pair: "{pair}", Interval: {interval} min'.format(
pair=pair,
interval=interval,
))
return False
return True
def download_backtesting_testdata(pair: str, interval: int = 5) -> bool:
"""
Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pairs: list of pairs to download
:return: bool
"""
path = testdata_path()
logger.info('Download the pair: "{pair}", Interval: {interval} min'.format(
pair=pair,
interval=interval,
))
filepair = pair.replace("-", "_")
filename = os.path.join(path, '{}-{}.json'.format(
filepair,
interval,
))
filename = filename.replace('USDT_BTC', 'BTC_FAKEBULL')
if os.path.isfile(filename):
with open(filename, "rt") as fp:
data = json.load(fp)
logger.debug("Current Start:", data[1]['T'])
logger.debug("Current End: ", data[-1:][0]['T'])
else:
data = []
logger.debug("Current Start: None")
logger.debug("Current End: None")
new_data = get_ticker_history(pair = pair, tick_interval = int(interval))
for row in new_data:
if row not in data:
data.append(row)
logger.debug("New Start:", data[1]['T'])
logger.debug("New End: ", data[-1:][0]['T'])
data = sorted(data, key=lambda data: data['T'])
with open(filename, "wt") as fp:
json.dump(data, fp)
return True

View File

@ -132,13 +132,14 @@ def start(args):
logger.info('Using ticker_interval: %s ...', args.ticker_interval) logger.info('Using ticker_interval: %s ...', args.ticker_interval)
data = {} data = {}
pairs = config['exchange']['pair_whitelist']
if args.live: if args.live:
logger.info('Downloading data for all pairs in whitelist ...') logger.info('Downloading data for all pairs in whitelist ...')
for pair in config['exchange']['pair_whitelist']: for pair in pairs:
data[pair] = exchange.get_ticker_history(pair, args.ticker_interval) data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
else: else:
logger.info('Using local backtesting data (ignoring whitelist in given config) ...') logger.info('Using local backtesting data (using whitelist in given config) ...')
data = load_data(args.ticker_interval) data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs)
logger.info('Using stake_currency: %s ...', config['stake_currency']) logger.info('Using stake_currency: %s ...', config['stake_currency'])
logger.info('Using stake_amount: %s ...', config['stake_amount']) logger.info('Using stake_amount: %s ...', config['stake_amount'])

View File

@ -14,6 +14,7 @@ from pandas import DataFrame
from freqtrade import exchange, optimize from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex from freqtrade.exchange import Bittrex
from freqtrade.misc import load_config
from freqtrade.optimize.backtesting import backtest from freqtrade.optimize.backtesting import backtest
from freqtrade.vendor.qtpylib.indicators import crossed_above from freqtrade.vendor.qtpylib.indicators import crossed_above
@ -34,7 +35,7 @@ AVG_PROFIT_TO_BEAT = 0.2
AVG_DURATION_TO_BEAT = 50 AVG_DURATION_TO_BEAT = 50
# Configuration and data used by hyperopt # Configuration and data used by hyperopt
PROCESSED = optimize.preprocess(optimize.load_data()) PROCESSED = []
OPTIMIZE_CONFIG = { OPTIMIZE_CONFIG = {
'max_open_trades': 3, 'max_open_trades': 3,
'stake_currency': 'BTC', 'stake_currency': 'BTC',
@ -215,7 +216,7 @@ def buy_strategy_generator(params):
def start(args): def start(args):
global TOTAL_TRIES global TOTAL_TRIES, PROCESSED
TOTAL_TRIES = args.epochs TOTAL_TRIES = args.epochs
exchange._API = Bittrex({'key': '', 'secret': ''}) exchange._API = Bittrex({'key': '', 'secret': ''})
@ -226,6 +227,11 @@ def start(args):
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
) )
logger.info('Using config: %s ...', args.config)
config = load_config(args.config)
pairs = config['exchange']['pair_whitelist']
PROCESSED = optimize.preprocess(optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval))
if args.mongodb: if args.mongodb:
logger.info('Using mongodb ...') logger.info('Using mongodb ...')
logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')

View File

@ -95,7 +95,12 @@ def test_parse_args_backtesting_invalid():
def test_parse_args_backtesting_custom(mocker): def test_parse_args_backtesting_custom(mocker):
backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock()) backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock())
args = parse_args(['-c', 'test_conf.json', 'backtesting', '--live', '--ticker-interval', '1']) args = parse_args([
'-c', 'test_conf.json',
'backtesting',
'--live',
'--ticker-interval', '1',
'--refresh-pairs-cached'])
assert args is None assert args is None
assert backtesting_mock.call_count == 1 assert backtesting_mock.call_count == 1
@ -106,6 +111,7 @@ def test_parse_args_backtesting_custom(mocker):
assert call_args.subparser == 'backtesting' assert call_args.subparser == 'backtesting'
assert call_args.func is not None assert call_args.func is not None
assert call_args.ticker_interval == 1 assert call_args.ticker_interval == 1
assert call_args.refresh_pairs is True
def test_parse_args_hyperopt(mocker): def test_parse_args_hyperopt(mocker):

View File

@ -4,6 +4,8 @@
from freqtrade import exchange, optimize from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex from freqtrade.exchange import Bittrex
from freqtrade.optimize.backtesting import backtest from freqtrade.optimize.backtesting import backtest
from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata
import os
import pytest import pytest
@ -13,8 +15,8 @@ def test_backtest(default_conf, mocker):
data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH'])
results = backtest(default_conf, optimize.preprocess(data), 10, True) results = backtest(default_conf, optimize.preprocess(data), 10, True)
num_resutls = len(results) num_results = len(results)
assert num_resutls > 0 assert num_results > 0
def test_1min_ticker_interval(default_conf, mocker): def test_1min_ticker_interval(default_conf, mocker):
@ -26,7 +28,69 @@ def test_1min_ticker_interval(default_conf, mocker):
results = backtest(default_conf, optimize.preprocess(data), 1, True) results = backtest(default_conf, optimize.preprocess(data), 1, True)
assert len(results) > 0 assert len(results) > 0
# Run a backtesting for 5min ticker_interval def test_backtest_with_new_pair(default_conf, mocker):
with pytest.raises(FileNotFoundError): mocker.patch.dict('freqtrade.main._CONF', default_conf)
data = optimize.load_data(ticker_interval=5, pairs=['BTC_UNITEST']) exchange._API = Bittrex({'key': '', 'secret': ''})
results = backtest(default_conf, optimize.preprocess(data), 1, True)
optimize.load_data(ticker_interval=1, pairs=['BTC_MEME'])
file = 'freqtrade/tests/testdata/BTC_MEME-1.json'
assert os.path.isfile(file) is True
# delete file freshly downloaded
if os.path.isfile(file):
os.remove(file)
def test_testdata_path():
assert str('freqtrade/optimize/../tests/testdata') in testdata_path()
def test_download_pairs(default_conf, mocker):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json'
file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json'
file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json'
file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json'
assert download_pairs(pairs = ['BTC-MEME', 'BTC-CFI']) is True
assert os.path.isfile(file1_1) is True
assert os.path.isfile(file1_5) is True
assert os.path.isfile(file2_1) is True
assert os.path.isfile(file2_5) is True
# delete files freshly downloaded
if os.path.isfile(file1_1):
os.remove(file1_1)
if os.path.isfile(file1_5):
os.remove(file1_5)
if os.path.isfile(file2_1):
os.remove(file2_1)
if os.path.isfile(file2_5):
os.remove(file2_5)
def test_download_backtesting_testdata(default_conf, mocker):
mocker.patch.dict('freqtrade.main._CONF', default_conf)
exchange._API = Bittrex({'key': '', 'secret': ''})
# Download a 1 min ticker file
file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json'
download_backtesting_testdata(pair = "BTC-XEL", interval = 1)
assert os.path.isfile(file1) is True
if os.path.isfile(file1):
os.remove(file1)
# Download a 5 min ticker file
file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json'
download_backtesting_testdata(pair = "BTC-STORJ", interval = 5)
assert os.path.isfile(file2) is True
if os.path.isfile(file2):
os.remove(file2)