Merge pull request #328 from kryofly/datadir

--datadir <path> argument

docs/backtesting.md

@@ -46,6 +46,11 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-c
 python3 ./freqtrade/main.py backtesting --realistic-simulation --live
 ```
 
+**Using a different on-disk ticker-data source**
+```bash
+python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101
+```
+
 For help about backtesting usage, please refer to 
 [Backtesting commands](#backtesting-commands).
 
@@ -101,4 +106,4 @@ strategies, your configuration, and the crypto-currency you have set up.
 ## Next step
 Great, your strategy is profitable. What if the bot can give your the
 optimal parameters to use for your strategy?  
-Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md)
+Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md)

docs/bot-usage.md

@@ -26,6 +26,9 @@ optional arguments:
                         specify configuration file (default: config.json)
   -v, --verbose         be verbose
   --version             show program's version number and exit
+  -dd PATH, --datadir PATH
+                        Path is from where backtesting and hyperopt will load the
+                        ticker data files (default freqdata/tests/testdata).
   --dynamic-whitelist [INT]
                         dynamically generate and update whitelist based on 24h
                         BaseVolume (Default 20 currencies)
@@ -133,4 +136,4 @@ in [misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L
 ## Next step
 The optimal strategy of the bot will change with time depending of the
 market trends. The next step is to 
-[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md).
+[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md).

docs/hyperopt.md

@@ -163,6 +163,11 @@ We strongly recommend to use `screen` to prevent any connection loss.
 python3 ./freqtrade/main.py -c config.json hyperopt
 ```
 
+### Execute hyperopt with different ticker-data source
+If you would like to learn parameters using an alternate ticke-data that
+you have on-disk, use the --datadir PATH option. Default hyperopt will
+use data from directory freqtrade/tests/testdata.
+
 ### Hyperopt with MongoDB
 Hyperopt with MongoDB, is like Hyperopt under steroids. As you saw by
 executing the previous command is the execution takes a long time. 

freqtrade/misc.py

@@ -125,6 +125,14 @@ def parse_args(args: List[str], description: str):
         action='store_true',
         dest='dry_run_db',
     )
+    parser.add_argument(
+        '-dd', '--datadir',
+        help='path to backtest data (default freqdata/tests/testdata',
+        dest='datadir',
+        default='freqtrade/tests/testdata',
+        type=str,
+        metavar='PATH',
+    )
     parser.add_argument(
         '--dynamic-whitelist',
         help='dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies)',  # noqa

freqtrade/optimize/__init__.py

@@ -12,12 +12,12 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
 logger = logging.getLogger(__name__)
 
 
-def load_tickerdata_file(pair, ticker_interval):
+def load_tickerdata_file(datadir, pair, ticker_interval):
     """
     Load a pair from file,
     :return dict OR empty if unsuccesful
     """
-    path = testdata_path()
+    path = make_testdata_path(datadir)
     file = '{abspath}/{pair}-{ticker_interval}.json'.format(
         abspath=path,
         pair=pair,
@@ -33,7 +33,7 @@ def load_tickerdata_file(pair, ticker_interval):
     return pairdata
 
 
-def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None,
+def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]] = None,
               refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
     """
     Loads ticker history data for the given parameters
@@ -47,16 +47,16 @@ def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None,
 
     # If the user force the refresh of pairs
     if refresh_pairs:
-        logger.info('Download data for all pairs and store them in freqtrade/tests/testsdata')
+        logger.info('Download data for all pairs and store them in %s', datadir)
-        download_pairs(_pairs)
+        download_pairs(datadir, _pairs)
 
     for pair in _pairs:
-        pairdata = load_tickerdata_file(pair, ticker_interval)
+        pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
         if not pairdata:
             # download the tickerdata from exchange
-            download_backtesting_testdata(pair=pair, interval=ticker_interval)
+            download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
             # and retry reading the pair
-            pairdata = load_tickerdata_file(pair, ticker_interval)
+            pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
         result[pair] = pairdata
     return result
 
@@ -67,17 +67,18 @@ def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
             for pair, pair_data in tickerdata.items()}
 
 
-def testdata_path() -> str:
+def make_testdata_path(datadir: str) -> str:
     """Return the path where testdata files are stored"""
-    return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'tests', 'testdata'))
+    return datadir or os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                   '..', 'tests', 'testdata'))
 
 
-def download_pairs(pairs: List[str]) -> bool:
+def download_pairs(datadir, pairs: List[str]) -> bool:
     """For each pairs passed in parameters, download 1 and 5 ticker intervals"""
     for pair in pairs:
         try:
             for interval in [1, 5]:
-                download_backtesting_testdata(pair=pair, interval=interval)
+                download_backtesting_testdata(datadir, pair=pair, interval=interval)
         except BaseException:
             logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format(
                 pair=pair,
@@ -87,7 +88,7 @@ def download_pairs(pairs: List[str]) -> bool:
     return True
 
 
-def download_backtesting_testdata(pair: str, interval: int = 5) -> bool:
+def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool:
     """
     Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
     Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
@@ -95,7 +96,7 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool:
     :return: bool
     """
 
-    path = testdata_path()
+    path = make_testdata_path(datadir)
     logger.info('Download the pair: "{pair}", Interval: {interval} min'.format(
         pair=pair,
         interval=interval,

freqtrade/optimize/backtesting.py

@@ -162,7 +162,7 @@ def start(args):
             data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
         logger.info('Using local backtesting data (using whitelist in given config) ...')
-        data = optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval,
+        data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval,
                                   refresh_pairs=args.refresh_pairs)
 
         logger.info('Using stake_currency: %s ...', config['stake_currency'])

freqtrade/optimize/hyperopt.py

@@ -36,7 +36,7 @@ CURRENT_BEST_LOSS = 100
 EXPECTED_MAX_PROFIT = 3.85
 
 # Configuration and data used by hyperopt
-PROCESSED = optimize.preprocess(optimize.load_data())
+PROCESSED = None  # optimize.preprocess(optimize.load_data())
 OPTIMIZE_CONFIG = hyperopt_optimize_conf()
 
 # Monkey patch config
@@ -224,7 +224,7 @@ def start(args):
     config = load_config(args.config)
     pairs = config['exchange']['pair_whitelist']
     PROCESSED = optimize.preprocess(optimize.load_data(
-        pairs=pairs, ticker_interval=args.ticker_interval))
+        args.datadir, pairs=pairs, ticker_interval=args.ticker_interval))
 
     if args.mongodb:
         logger.info('Using mongodb ...')

freqtrade/tests/optimize/test_backtesting.py

@@ -32,7 +32,7 @@ def test_generate_text_table():
 
 def test_get_timeframe():
     data = preprocess(optimize.load_data(
-        ticker_interval=1, pairs=['BTC_UNITEST']))
+        None, ticker_interval=1, pairs=['BTC_UNITEST']))
     min_date, max_date = get_timeframe(data)
     assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
     assert max_date.isoformat() == '2017-11-14T22:59:00+00:00'
@@ -42,7 +42,7 @@ def test_backtest(default_conf, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH'])
+    data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH'])
     results = backtest(default_conf['stake_amount'],
                        optimize.preprocess(data), 10, True)
     assert not results.empty
@@ -53,7 +53,7 @@ def test_backtest_1min_ticker_interval(default_conf, mocker):
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
     # Run a backtesting for an exiting 5min ticker_interval
-    data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST'])
+    data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'])
     results = backtest(default_conf['stake_amount'],
                        optimize.preprocess(data), 1, True)
     assert not results.empty
@@ -67,7 +67,7 @@ def trim_dictlist(dl, num):
 
 
 def load_data_test(what):
-    data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST'])
+    data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'])
     data = trim_dictlist(data, -100)
     pair = data['BTC_UNITEST']
     datalen = len(pair)
@@ -124,7 +124,7 @@ def simple_backtest(config, contour, num_results):
 
 def test_backtest2(default_conf, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
-    data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH'])
+    data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH'])
     results = backtest(default_conf['stake_amount'],
                        optimize.preprocess(data), 10, True)
     assert not results.empty
@@ -149,8 +149,8 @@ def test_backtest_pricecontours(default_conf, mocker):
         simple_backtest(default_conf, contour, numres)
 
 
-def mocked_load_data(pairs=[], ticker_interval=0, refresh_pairs=False):
+def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False):
-    tickerdata = optimize.load_tickerdata_file('BTC_UNITEST', 1)
+    tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1)
     pairdata = {'BTC_UNITEST': tickerdata}
     return trim_dictlist(pairdata, -100)
 
@@ -165,6 +165,7 @@ def test_backtest_start(default_conf, mocker, caplog):
     args.ticker_interval = 1
     args.level = 10
     args.live = False
+    args.datadir = None
     backtesting.start(args)
     # check the logs, that will contain the backtest result
     exists = ['Using max_open_trades: 1 ...',

freqtrade/tests/optimize/test_optimize.py

@@ -5,7 +5,7 @@ import logging
 from shutil import copyfile
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
-from freqtrade.optimize.__init__ import testdata_path, download_pairs,\
+from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\
      download_backtesting_testdata, load_tickerdata_file
 
 # Change this if modifying BTC_UNITEST testdatafile
@@ -51,7 +51,7 @@ def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog):
 
     file = 'freqtrade/tests/testdata/BTC_ETH-5.json'
     _backup_file(file, copy_file=True)
-    optimize.load_data(pairs=['BTC_ETH'])
+    optimize.load_data(None, pairs=['BTC_ETH'])
     assert os.path.isfile(file) is True
     assert ('freqtrade.optimize',
             logging.INFO,
@@ -68,7 +68,7 @@ def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog):
 
     file = 'freqtrade/tests/testdata/BTC_ETH-1.json'
     _backup_file(file, copy_file=True)
-    optimize.load_data(ticker_interval=1, pairs=['BTC_ETH'])
+    optimize.load_data(None, ticker_interval=1, pairs=['BTC_ETH'])
     assert os.path.isfile(file) is True
     assert ('freqtrade.optimize',
             logging.INFO,
@@ -85,7 +85,7 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl
 
     file = 'freqtrade/tests/testdata/BTC_MEME-1.json'
     _backup_file(file)
-    optimize.load_data(ticker_interval=1, pairs=['BTC_MEME'])
+    optimize.load_data(None, ticker_interval=1, pairs=['BTC_MEME'])
     assert os.path.isfile(file) is True
     assert ('freqtrade.optimize',
             logging.INFO,
@@ -95,7 +95,7 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl
 
 
 def test_testdata_path():
-    assert os.path.join('freqtrade', 'tests', 'testdata') in testdata_path()
+    assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None)
 
 
 def test_download_pairs(default_conf, ticker_history, mocker):
@@ -113,7 +113,7 @@ def test_download_pairs(default_conf, ticker_history, mocker):
     _backup_file(file2_1)
     _backup_file(file2_5)
 
-    assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True
+    assert download_pairs(None, pairs=['BTC-MEME', 'BTC-CFI']) is True
 
     assert os.path.isfile(file1_1) is True
     assert os.path.isfile(file1_5) is True
@@ -139,7 +139,7 @@ def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog):
     _backup_file(file1_1)
     _backup_file(file1_5)
 
-    download_pairs(pairs=['BTC-MEME'])
+    download_pairs(None, pairs=['BTC-MEME'])
     # clean files freshly downloaded
     _clean_test_file(file1_1)
     _clean_test_file(file1_5)
@@ -157,7 +157,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
     # Download a 1 min ticker file
     file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json'
     _backup_file(file1)
-    download_backtesting_testdata(pair="BTC-XEL", interval=1)
+    download_backtesting_testdata(None, pair="BTC-XEL", interval=1)
     assert os.path.isfile(file1) is True
     _clean_test_file(file1)
 
@@ -165,12 +165,12 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker):
     file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json'
     _backup_file(file2)
 
-    download_backtesting_testdata(pair="BTC-STORJ", interval=5)
+    download_backtesting_testdata(None, pair="BTC-STORJ", interval=5)
     assert os.path.isfile(file2) is True
     _clean_test_file(file2)
 
 
 def test_load_tickerdata_file():
-    assert not load_tickerdata_file('BTC_UNITEST', 7)
+    assert not load_tickerdata_file(None, 'BTC_UNITEST', 7)
-    tickerdata = load_tickerdata_file('BTC_UNITEST', 1)
+    tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 1)
     assert _btc_unittest_length == len(tickerdata)

freqtrade/tests/test_dataframe.py

@@ -7,7 +7,7 @@ _pairs = ['BTC_ETH']
 
 
 def load_dataframe_pair(pairs):
-    ld = freqtrade.optimize.load_data(ticker_interval=5, pairs=pairs)
+    ld = freqtrade.optimize.load_data(None, ticker_interval=5, pairs=pairs)
     assert isinstance(ld, dict)
     assert isinstance(pairs[0], str)
     dataframe = ld[pairs[0]]