Allow user to update testdata files with parameter --refresh-pairs-cached (#174)

.gitignore

@@ -1,3 +1,6 @@
+# Freqtrade rules
+freqtrade/tests/testdata/*.json
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -192,6 +192,7 @@ Backtesting also uses the config specified via `-c/--config`.
 
 ```
 usage: freqtrade backtesting [-h] [-l] [-i INT] [--realistic-simulation]
+                             [-r]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -201,9 +202,25 @@ optional arguments:
   --realistic-simulation
                         uses max_open_trades from config to simulate real
                         world limitations
-
+  -r, --refresh-pairs-cached
+                        refresh the pairs files in tests/testdata with 
+                        the latest data from Bittrex. Use it if you want
+                        to run your backtesting with up-to-date data.
 ```
 
+#### How to use --refresh-pairs-cached parameter?
+The first time your run Backtesting, it will take the pairs your have 
+set in your config file and download data from Bittrex. 
+
+If for any reason you want to update your data set, you use 
+`--refresh-pairs-cached` to force Backtesting to update the data it has. 
+**Use it only if you want to update your data set. You will not be able
+to come back to the previous version.**
+
+To test your strategy with latest data, we recommend to continue using  
+the parameter `-l` or `--live`.
+
+
 ### Hyperopt
 
 It is possible to use hyperopt for trading strategy optimization.

freqtrade/misc.py

@@ -166,6 +166,13 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
         action='store_true',
         dest='realistic_simulation',
     )
+    backtesting_cmd.add_argument(
+        '-r', '--refresh-pairs-cached',
+        help='refresh the pairs files in tests/testdata with the latest data from Bittrex. Use it if you want to \
+              run your backtesting with up-to-date data.',
+        action='store_true',
+        dest='refresh_pairs',
+    )
 
     # Add hyperopt subcommand
     hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')

freqtrade/optimize/__init__.py

@@ -1,34 +1,46 @@
 # pragma pylint: disable=missing-docstring
 
-
+import logging
 import json
 import os
 from typing import Optional, List, Dict
+import time
+from freqtrade.exchange import get_ticker_history
 
 from pandas import DataFrame
 
 from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
 
+logger = logging.getLogger(__name__)
 
-def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None) -> Dict[str, List]:
+
+def load_data(pairs: List[str], ticker_interval: int = 5, refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
     """
     Loads ticker history data for the given parameters
     :param ticker_interval: ticker interval in minutes
     :param pairs: list of pairs
     :return: dict
     """
-    path = os.path.abspath(os.path.dirname(__file__))
+    path = testdata_path()
     result = {}
-    _pairs = pairs or [
-        "BTC_ETH", "BTC_LTC", "BTC_ETC", "BTC_DASH", "BTC_ZEC",
-        "BTC_XLM", "BTC_NXT", "BTC_POWR", "BTC_ADA", "BTC_XMR",
-    ]
-    for pair in _pairs:
-        with open('{abspath}/../tests/testdata/{pair}-{ticker_interval}.json'.format(
+
+    # If the user force the refresh of pairs
+    if refresh_pairs:
+        logger.info('Download data for all pairs and store them in freqtrade/tests/testsdata')
+        download_pairs(pairs)
+
+    for pair in pairs:
+        file = '{abspath}/{pair}-{ticker_interval}.json'.format(
             abspath=path,
             pair=pair,
             ticker_interval=ticker_interval,
-        )) as tickerdata:
+        )
+        # The file does not exist we download it
+        if not os.path.isfile(file):
+            download_backtesting_testdata(pair=pair, interval=ticker_interval)
+
+        # Read the file, load the json
+        with open(file) as tickerdata:
             result[pair] = json.load(tickerdata)
     return result
 
@@ -39,3 +51,68 @@ def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
     for pair, pair_data in tickerdata.items():
         processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data))
     return processed
+
+
+def testdata_path() -> str:
+    """Return the path where testdata files are stored"""
+    return os.path.abspath(os.path.dirname(__file__)) + '/../tests/testdata'
+
+
+def download_pairs(pairs: List[str]) -> bool:
+    """For each pairs passed in parameters, download 1 and 5 ticker intervals"""
+    for pair in pairs:
+        try:
+            for interval in [1,5]:
+                download_backtesting_testdata(pair=pair, interval=interval)
+        except BaseException:
+            logger.info('Impossible to download the pair: "{pair}", Interval: {interval} min'.format(
+                pair=pair,
+                interval=interval,
+            ))
+            return False
+    return True
+
+
+def download_backtesting_testdata(pair: str, interval: int = 5) -> bool:
+    """
+    Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
+    Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
+    :param pairs: list of pairs to download
+    :return: bool
+    """
+
+    path = testdata_path()
+    logger.info('Download the pair: "{pair}", Interval: {interval} min'.format(
+        pair=pair,
+        interval=interval,
+    ))
+
+    filepair = pair.replace("-", "_")
+    filename = os.path.join(path, '{}-{}.json'.format(
+        filepair,
+        interval,
+    ))
+    filename = filename.replace('USDT_BTC', 'BTC_FAKEBULL')
+
+    if os.path.isfile(filename):
+        with open(filename, "rt") as fp:
+            data = json.load(fp)
+        logger.debug("Current Start:", data[1]['T'])
+        logger.debug("Current End: ", data[-1:][0]['T'])
+    else:
+        data = []
+        logger.debug("Current Start: None")
+        logger.debug("Current End: None")
+
+    new_data = get_ticker_history(pair = pair, tick_interval = int(interval))
+    for row in new_data:
+        if row not in data:
+            data.append(row)
+    logger.debug("New Start:", data[1]['T'])
+    logger.debug("New End: ", data[-1:][0]['T'])
+    data = sorted(data, key=lambda data: data['T'])
+
+    with open(filename, "wt") as fp:
+        json.dump(data, fp)
+
+    return True

freqtrade/optimize/backtesting.py

@@ -132,13 +132,14 @@ def start(args):
     logger.info('Using ticker_interval: %s ...', args.ticker_interval)
 
     data = {}
+    pairs = config['exchange']['pair_whitelist']
     if args.live:
         logger.info('Downloading data for all pairs in whitelist ...')
-        for pair in config['exchange']['pair_whitelist']:
+        for pair in pairs:
             data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
-        logger.info('Using local backtesting data (ignoring whitelist in given config) ...')
-        data = load_data(args.ticker_interval)
+        logger.info('Using local backtesting data (using whitelist in given config) ...')
+        data = load_data(pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs)
 
         logger.info('Using stake_currency: %s ...', config['stake_currency'])
         logger.info('Using stake_amount: %s ...', config['stake_amount'])

freqtrade/optimize/hyperopt.py

@@ -14,6 +14,7 @@ from pandas import DataFrame
 
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
+from freqtrade.misc import load_config
 from freqtrade.optimize.backtesting import backtest
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
@@ -34,7 +35,7 @@ AVG_PROFIT_TO_BEAT = 0.2
 AVG_DURATION_TO_BEAT = 50
 
 # Configuration and data used by hyperopt
-PROCESSED = optimize.preprocess(optimize.load_data())
+PROCESSED = []
 OPTIMIZE_CONFIG = {
     'max_open_trades': 3,
     'stake_currency': 'BTC',
@@ -215,7 +216,7 @@ def buy_strategy_generator(params):
 
 
 def start(args):
-    global TOTAL_TRIES
+    global TOTAL_TRIES, PROCESSED
     TOTAL_TRIES = args.epochs
 
     exchange._API = Bittrex({'key': '', 'secret': ''})
@@ -226,6 +227,11 @@ def start(args):
         format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
     )
 
+    logger.info('Using config: %s ...', args.config)
+    config = load_config(args.config)
+    pairs = config['exchange']['pair_whitelist']
+    PROCESSED = optimize.preprocess(optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval))
+
     if args.mongodb:
         logger.info('Using mongodb ...')
         logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')

freqtrade/tests/test_misc.py

@@ -95,7 +95,12 @@ def test_parse_args_backtesting_invalid():
 
 def test_parse_args_backtesting_custom(mocker):
     backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock())
-    args = parse_args(['-c', 'test_conf.json', 'backtesting', '--live', '--ticker-interval', '1'])
+    args = parse_args([
+        '-c', 'test_conf.json',
+        'backtesting',
+        '--live',
+        '--ticker-interval', '1',
+        '--refresh-pairs-cached'])
     assert args is None
     assert backtesting_mock.call_count == 1
 
@@ -106,6 +111,7 @@ def test_parse_args_backtesting_custom(mocker):
     assert call_args.subparser == 'backtesting'
     assert call_args.func is not None
     assert call_args.ticker_interval == 1
+    assert call_args.refresh_pairs is True
 
 
 def test_parse_args_hyperopt(mocker):

freqtrade/tests/test_optimize_backtesting.py

@@ -4,6 +4,8 @@
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
 from freqtrade.optimize.backtesting import backtest
+from freqtrade.optimize.__init__ import testdata_path, download_pairs, download_backtesting_testdata
+import os
 
 import pytest
 
@@ -13,8 +15,8 @@ def test_backtest(default_conf, mocker):
 
     data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH'])
     results = backtest(default_conf, optimize.preprocess(data), 10, True)
-    num_resutls = len(results)
-    assert num_resutls > 0
+    num_results = len(results)
+    assert num_results > 0
 
 
 def test_1min_ticker_interval(default_conf, mocker):
@@ -26,7 +28,69 @@ def test_1min_ticker_interval(default_conf, mocker):
     results = backtest(default_conf, optimize.preprocess(data), 1, True)
     assert len(results) > 0
 
-    # Run a backtesting for 5min ticker_interval
-    with pytest.raises(FileNotFoundError):
-        data = optimize.load_data(ticker_interval=5, pairs=['BTC_UNITEST'])
-        results = backtest(default_conf, optimize.preprocess(data), 1, True)
+def test_backtest_with_new_pair(default_conf, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    exchange._API = Bittrex({'key': '', 'secret': ''})
+
+    optimize.load_data(ticker_interval=1, pairs=['BTC_MEME'])
+    file = 'freqtrade/tests/testdata/BTC_MEME-1.json'
+    assert os.path.isfile(file) is True
+
+    # delete file freshly downloaded
+    if os.path.isfile(file):
+        os.remove(file)
+
+
+def test_testdata_path():
+    assert str('freqtrade/optimize/../tests/testdata') in testdata_path()
+
+
+def test_download_pairs(default_conf, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    exchange._API = Bittrex({'key': '', 'secret': ''})
+
+    file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json'
+    file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json'
+    file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json'
+    file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json'
+
+    assert download_pairs(pairs = ['BTC-MEME', 'BTC-CFI']) is True
+
+    assert os.path.isfile(file1_1) is True
+    assert os.path.isfile(file1_5) is True
+    assert os.path.isfile(file2_1) is True
+    assert os.path.isfile(file2_5) is True
+
+    # delete files freshly downloaded
+    if os.path.isfile(file1_1):
+        os.remove(file1_1)
+
+    if os.path.isfile(file1_5):
+        os.remove(file1_5)
+
+    if os.path.isfile(file2_1):
+        os.remove(file2_1)
+
+    if os.path.isfile(file2_5):
+        os.remove(file2_5)
+
+
+def test_download_backtesting_testdata(default_conf, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    exchange._API = Bittrex({'key': '', 'secret': ''})
+
+    # Download a 1 min ticker file
+    file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json'
+    download_backtesting_testdata(pair = "BTC-XEL", interval = 1)
+    assert os.path.isfile(file1) is True
+
+    if os.path.isfile(file1):
+        os.remove(file1)
+
+    # Download a 5 min ticker file
+    file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json'
+    download_backtesting_testdata(pair = "BTC-STORJ", interval = 5)
+    assert os.path.isfile(file2) is True
+
+    if os.path.isfile(file2):
+        os.remove(file2)